Merge pull request #6 from Jibec/master

Use new helpers, add backup/restore and uwsgi

README.md

@@ -1,21 +1,18 @@
 # Mozilla’s Sync Server for Yunohost
 
-The Sync Server provides a replacement for Firefox’s default server (hosted at Mozilla). Its code is available on [the Sync-1.5 Server documentation](https://docs.services.mozilla.com/howtos/run-sync-1.5.html).
+The Sync Server provides a replacement for Firefox’s default server (hosted at Mozilla).
 
 By default, a server set up will defer authentication to the Mozilla-hosted accounts server at [https://accounts.firefox.com](https://accounts.firefox.com). So you will still have to authenticate at Mozilla, but _the storage of your information will be done on your host_.
 
-**Shipped version:** 1.5
+**Shipped version:** 1.8
 
 ## Configuring
 
 Once installed, reaching `http://domain.tld/path` should show a page explaining how to configure it. Otherwise please refer to the [Yunohost page](https://yunohost.org/#/app_ffsync).
 
-## Contributing
+## Links
 
-From the `sources` directory, do as follows:
+ * Report a bug about this package: https://github.com/YunoHost-Apps/ffsync_ynh
-
+ * Report a bug about Funkwhale itself: https://github.com/mozilla-services/syncserver
-`make build`
+ * Documentation: https://docs.services.mozilla.com/howtos/run-sync-1.5.html
-
+ * YunoHost website: https://yunohost.org/
-`make test`
-
-`./local/bin/gunicorn --reload --paste syncserver.ini` instead of the classical `make serve` to take into account changes you will surely do by developping the app.

check_process

@@ -0,0 +1,41 @@
+;; Test name
+	; Manifest
+		domain="domain.tld"	(DOMAIN)
+		path="/path"	(PATH)
+	; Checks
+		pkg_linter=1
+		setup_sub_dir=1
+		setup_root=1
+		setup_nourl=0
+		setup_private=0
+		setup_public=1
+		upgrade=1
+		upgrade=1	from_commit=fd6350495d5a1d864ae30e1a61e18939fdb6a428
+		upgrade=1	from_commit=267ccc21f7b52d22bc3d5b9cd6239857b9a82aad
+		backup_restore=1
+		multi_instance=1
+		incorrect_path=1
+		port_already_use=0
+		change_url=0
+;;; Levels
+	Level 1=auto
+	Level 2=auto
+	Level 3=auto
+	# impossible with Firefox Sync (it uses Firefox Accounts)
+	Level 4=na
+	Level 5=auto
+	Level 6=auto
+	Level 7=auto
+	Level 8=0
+	Level 9=0
+	Level 10=0
+;;; Options
+Email=jean-baptiste@holcroft.fr
+Notification=fail
+;;; Upgrade options
+	; commit=fd6350495d5a1d864ae30e1a61e18939fdb6a428
+		name=latest published version
+		manifest_arg=domain=DOMAIN&path=PATH
+	; commit=267ccc21f7b52d22bc3d5b9cd6239857b9a82aad
+		name=latest git commit before Jibec's rewriting
+		manifest_arg=domain=DOMAIN&path=PATH

conf/app.src

@@ -0,0 +1,4 @@
+SOURCE_URL=https://github.com/mozilla-services/syncserver/archive/1.8.0.tar.gz
+SOURCE_SUM=728206bcffec7a305e97e7cd4d465b3fa56f39f8e1fd55d98e49a866016d61e0
+SOURCE_SUM_PRG=sha256sum
+SOURCE_FORMAT=tar.gz

conf/ffsync

@@ -1,75 +0,0 @@
-#!/bin/bash
-# /etc/init.d/sync
-# version 0.1 2013-03-12 (YYYY-MM-DD)
-
-### BEGIN INIT INFO
-# Provides:   sync
-# Required-Start: $local_fs $remote_fs
-# Required-Stop:  $local_fs $remote_fs
-# Should-Start:   $network
-# Should-Stop:    $network
-# Default-Start:  2 3 4 5
-# Default-Stop:   0 1 6
-# Short-Description:    Mozilla Sync server
-# Description:    Starts the mozilla sync server
-### END INIT INFO
-
-# Source function library.
-#. /etc/rc.d/init.d/functions
-
-prog=sync
-SYNC_USER=ffsync
-SYNC_HOME=/opt/yunohost/ffsync
-CPU_COUNT=2
-pidfile=/tmp/sync.pid
-lockfile=/var/run/sync.lock
-conffile=${SYNC_HOME}/syncserver.ini
-GUNICORN=${SYNC_HOME}/local/bin/gunicorn
-GUNICORN_ARGS="--paste  $conffile --access-logfile /var/log/ffsync.log --daemon -p $pidfile"
-
-start () {
-    echo -n "Starting $prog"
-    start-stop-daemon --start -c ${SYNC_USER} --exec $GUNICORN -- $GUNICORN_ARGS
-    RETVAL=$?
-    echo
-    [ $RETVAL = 0 ] && touch ${lockfile}
-    return $RETVAL
-}
-
-stop() {
-    echo "Stopping $prog"
-    start-stop-daemon --stop --quiet --oknodo --pidfile ${pidfile}
-    #log_end_msg $?
-    rm -f ${pidfile}
-}
-
-status(){
-        if [[ -f ${pidfile} ]]; then
-                echo "Status: running."
-                exit 0;
-        else
-                echo "Status: not running."
-                exit 1;
-        fi
-}
-
-case "$1" in
-    start)
-        start
-        ;;
-    stop)
-        stop
-        ;;
-    status)
-        status
-        ;;
-    restart)
-        stop
-        start
-        ;;
-    *)
-        echo $"Usage: $prog {start|stop|restart|help}"
-        RETVAL=2
-esac
-
-exit $RETVAL

conf/ffsync.logrotate

@@ -1,9 +0,0 @@
-"/var/log/ffsync.log" {
-  copytruncate
-  daily
-  rotate 7
-  compress
-  delaycompress
-  missingok
-  notifempty
-}

conf/nginx.conf

@@ -1,15 +1,17 @@
-location PATHTOCHANGE {
+#sub_path_only rewrite ^__PATH__$ __PATH__/ permanent;
+
+location __PATH__/ {
+	# Path to source
+	alias __FINALPATH__/ ;
+
 	if ($scheme = http) {
 		rewrite ^ https://$server_name$request_uri? permanent;
 	}
-        proxy_set_header Host $http_host;
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Real-IP $remote_addr;
-        proxy_redirect off;
-        proxy_read_timeout 120;
-        proxy_connect_timeout 10;
-        proxy_pass http://127.0.0.1:5000/;
 
-        include conf.d/yunohost_panel.conf.inc;
+	include uwsgi_params;
+	# Needed for long running operations in admin interface
+	uwsgi_read_timeout 3600;
+	uwsgi_param SCRIPT_NAME __PATH__;
+	uwsgi_modifier1 30;
+	uwsgi_pass unix:///var/run/__NAME__/app.socket;
 }

conf/syncserver.ini

@@ -1,45 +0,0 @@
-[server:main]
-use = egg:gunicorn
-host = 0.0.0.0
-port = 5000
-workers = 1
-timeout = 30
-
-[app:main]
-use = egg:syncserver
-
-[syncserver]
-# This must be edited to point to the public URL of your server,
-# i.e. the URL as seen by Firefox.
-public_url = https://ynhbaseurl/
-
-# This defines the database in which to store all server data.
-sqluri = pymysql://yunouser:yunopass@localhost/yunobase
-
-# This is a secret key used for signing authentication tokens.
-# It should be long and randomly-generated.
-# The following command will give a suitable value on *nix systems:
-#
-#    head -c 20 /dev/urandom | sha1sum
-#
-# If not specified then the server will generate a temporary one at startup.
-secret = changesecret
-
-# Set this to "false" to disable new-user signups on the server.
-# Only request by existing accounts will be honoured.
-allow_new_users = true
-
-# Set this to "true" to work around a mismatch between public_url and
-# the application URL as seen by python, which can happen in certain reverse-
-# proxy hosting setups.  It will overwrite the WSGI environ dict with the
-# details from public_url.  This could have security implications if e.g.
-# you tell the app that it's on HTTPS but it's really on HTTP, so it should
-# only be used as a last resort and after careful checking of server config.
-force_wsgi_environ = true
-
-# Uncomment and edit the following to use a local BrowserID verifier
-# rather than posting assertions to the mozilla-hosted verifier.
-# Audiences should be set to your public_url without a trailing slash.
-#[browserid]
-#backend = tokenserver.verifiers.LocalVerifier
-#audiences = https://localhost:5000

conf/uwsgi-app@.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=%i uWSGI app
+After=syslog.target
+
+[Service]
+RuntimeDirectory=%i
+ExecStart=/usr/bin/uwsgi \
+        --ini /etc/uwsgi/apps-available/%i.ini \
+        --socket /var/run/%i/app.socket \
+        --logto /var/log/uwsgi/%i/%i.log
+User=%i
+Group=www-data
+Restart=on-failure
+KillSignal=SIGQUIT
+Type=notify
+StandardError=syslog
+NotifyAccess=all
+
+[Install]
+WantedBy=multi-user.target

conf/uwsgi.ini

@@ -1,9 +1,14 @@
-[server:main]
+[uwsgi]
-use = egg:gunicorn
+plugins       = python
-host = 0.0.0.0
+master        = true
-port = 5000
+protocol      = uwsgi
-workers = 1
+socket        = /var/run/__APP__/app.socket
-timeout = 30
+chmod-socket  = 660
+virtualenv    = __FINALPATH__/local
+wsgi-file     = __FINALPATH__/syncserver.wsgi
+python-path   = __FINALPATH__/local
+enable-threads = true
+close-on-exec  = true
 
 [app:main]
 use = egg:syncserver
@@ -11,10 +16,10 @@ use = egg:syncserver
 [syncserver]
 # This must be edited to point to the public URL of your server,
 # i.e. the URL as seen by Firefox.
-public_url = http://localhost:5000/
+public_url = https://__DOMAIN____PATH__
 
 # This defines the database in which to store all server data.
-#sqluri = sqlite:////tmp/syncserver.db
+sqluri = pymysql://__DB_USER__:__DB_PWD__@localhost/__DB_NAME__
 
 # This is a secret key used for signing authentication tokens.
 # It should be long and randomly-generated.
@@ -23,11 +28,11 @@ public_url = http://localhost:5000/
 #    head -c 20 /dev/urandom | sha1sum
 #
 # If not specified then the server will generate a temporary one at startup.
-#secret = INSERT_SECRET_KEY_HERE
+secret = __SECRET__
 
 # Set this to "false" to disable new-user signups on the server.
 # Only request by existing accounts will be honoured.
-# allow_new_users = false
+allow_new_users = true
 
 # Set this to "true" to work around a mismatch between public_url and
 # the application URL as seen by python, which can happen in certain reverse-
@@ -42,4 +47,3 @@ force_wsgi_environ = false
 # Audiences should be set to your public_url without a trailing slash.
 #[browserid]
 #backend = tokenserver.verifiers.LocalVerifier
-#audiences = https://localhost:5000

manifest.json

@@ -1,25 +1,31 @@
 {
     "name": "Firefox Sync Server",
     "id": "ffsync",
-    "url": "https://github.com/abeudin/ffsync_ynh",
+    "version": "1.8.0~ynh1",
+    "url": "https://github.com/mozilla-services/syncserver",
     "packaging_format": 1,
     "description": {
         "en": "Mozilla’s Sync-Server to host your Firefox account data",
         "fr": "Le serveur de synchronisation de Mozilla, pour héberger vos données Firefox"
     },
     "maintainer": {
+        "name": "Josué Tille",
+        "email": "josue@tille.ch"
+    },
+    "previous_maintainers": [{
         "name": "beudbeud",
         "email": "beudbeud@beudibox.fr",
-        "url": "https://github.com/balu-/FSyncMS"
+        "name": "jibec",
-    },
+        "email": "jean-baptiste@holcroft.fr"
+    }],
     "requirements": {
-        "yunohost": ">> 2.4.0"
+        "yunohost": ">=3.0.0"
 	},
-	"license": "free",
+	"license": "MPL-2.0",
 	"services": [
         "nginx"
 	],
-    "multi_instance": false,
+    "multi_instance": true,
     "arguments": {
         "install" : [
             {

scripts/_common.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+# Check if system wide templates are available and correcly configured
+#
+# usage: ynh_check_global_uwsgi_config
+ynh_check_global_uwsgi_config () {
+	uwsgi --version || ynh_die "You need to add uwsgi (and appropriate plugin) as a dependency"
+
+	cp ../conf/uwsgi-app@.service /etc/systemd/system/uwsgi-app@.service
+
+	systemctl daemon-reload
+}
+
+# Create a dedicated uwsgi ini file to use with generic uwsgi service
+#
+# This will use a template in ../conf/uwsgi.ini
+# and will replace the following keywords with
+# global variables that should be defined before calling
+# this helper :
+#
+#   __APP__       by  $app
+#   __PATH__      by  $path_url
+#   __FINALPATH__ by  $final_path
+#
+#  And dynamic variables (from the last example) :
+#   __PATH_2__    by $path_2
+#   __PORT_2__    by $port_2
+#
+# usage: ynh_add_uwsgi_service
+#
+# to interact with your service: `systemctl <action> uwsgi-app@app`
+ynh_add_uwsgi_service () {
+	ynh_check_global_uwsgi_config
+
+	local others_var=${1:-}
+	local finaluwsgiini="/etc/uwsgi/apps-available/$app.ini"
+
+	# www-data group is needed since it is this nginx who will start the service
+	usermod --append --groups www-data "$app" || ynh_die "It wasn't possible to add user $app to group www-data"
+
+	ynh_backup_if_checksum_is_different "$finaluwsgiini"
+	cp ../conf/uwsgi.ini "$finaluwsgiini"
+
+	# To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable.
+	# Substitute in a nginx config file only if the variable is not empty
+	if test -n "${final_path:-}"; then
+		ynh_replace_string "__FINALPATH__" "$final_path" "$finaluwsgiini"
+	fi
+	if test -n "${path_url:-}"; then
+		ynh_replace_string "__PATH__" "$path_url" "$finaluwsgiini"
+	fi
+	if test -n "${app:-}"; then
+		ynh_replace_string "__APP__" "$app" "$finaluwsgiini"
+	fi
+
+	# Replace all other variable given as arguments
+	for var_to_replace in $others_var
+	do
+		# ${var_to_replace^^} make the content of the variable on upper-cases
+		# ${!var_to_replace} get the content of the variable named $var_to_replace 
+		ynh_replace_string "__${var_to_replace^^}__" "${!var_to_replace}" "$finaluwsgiini"
+	done
+
+	ynh_store_file_checksum "$finaluwsgiini"
+
+	chown $app:root "$finaluwsgiini"
+
+	# make sure the folder for logs exists and set authorizations
+	mkdir -p /var/log/uwsgi/$app
+	chown $app:root /var/log/uwsgi/$app
+	chmod -R u=rwX,g=rX,o= /var/log/uwsgi/$app
+
+	systemctl daemon-reload
+	systemctl stop "uwsgi-app@$app.service"
+	systemctl enable "uwsgi-app@$app.service"
+	systemctl start "uwsgi-app@$app.service"
+
+	# Add as a service
+	yunohost service add "uwsgi-app@$app" --log "/var/log/uwsgi/$app/$app.log"
+}
+
+# Remove the dedicated uwsgi ini file
+#
+# usage: ynh_remove_uwsgi_service
+ynh_remove_uwsgi_service () {
+	local finaluwsgiini="/etc/uwsgi/apps-available/$app.ini"
+	if [ -e "$finaluwsgiini" ]; then
+		systemctl stop "uwsgi-app@$app.service"
+		systemctl disable "uwsgi-app@$app.service"
+		yunohost service remove "uwsgi-app@$app"
+
+		ynh_secure_remove "$finaluwsgiini"
+		ynh_secure_remove "/var/log/uwsgi/$app"
+	fi
+}

scripts/backup

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+#=================================================
+# GENERIC START
+#=================================================
+# IMPORT GENERIC HELPERS
+#=================================================
+
+source ../settings/scripts/_common.sh
+source /usr/share/yunohost/helpers
+
+#=================================================
+# MANAGE SCRIPT FAILURE
+#=================================================
+
+# Exit if an error occurs during the execution of the script
+ynh_abort_if_errors
+
+#=================================================
+# LOAD SETTINGS
+#=================================================
+
+app=$YNH_APP_INSTANCE_NAME
+
+final_path=$(ynh_app_setting_get "$app" final_path)
+domain=$(ynh_app_setting_get "$app" domain)
+db_name=$(ynh_app_setting_get "$app" db_name)
+
+#=================================================
+# STANDARD BACKUP STEPS
+#=================================================
+# BACKUP THE APP MAIN DIR
+#=================================================
+
+ynh_backup "$final_path"
+
+#=================================================
+# BACKUP THE NGINX CONFIGURATION
+#=================================================
+
+ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf"
+
+#=================================================
+# BACKUP THE MYSQL DATABASE
+#=================================================
+
+ynh_mysql_dump_db "$db_name" > db.sql
+
+#=================================================
+# SPECIFIC BACKUP
+#=================================================
+# Backup Log
+#=================================================
+
+ynh_backup "/var/log/uwsgi/$app"
+
+#=================================================
+# BACKUP THE UWSGI FILES
+#=================================================
+
+ynh_backup "/etc/uwsgi/apps-available/$app.ini"
+ynh_backup "/etc/systemd/system/uwsgi-app@.service"

scripts/install

@@ -1,105 +1,177 @@
 #!/bin/bash
 
-# Source app helpers
+#=================================================
-. /usr/share/yunohost/helpers
+# GENERIC START
+#=================================================
+# IMPORT GENERIC HELPERS
+#=================================================
+
+source _common.sh
+source /usr/share/yunohost/helpers
+
+#=================================================
+# MANAGE SCRIPT FAILURE
+#=================================================
+
+# Exit if an error occurs during the execution of the script
+ynh_abort_if_errors
+
+#=================================================
+# RETRIEVE ARGUMENTS FROM THE MANIFEST
+#=================================================
 
 # Retrieve arguments
-domain=$1
+domain=$YNH_APP_ARG_DOMAIN
-path=$2
+path_url=$YNH_APP_ARG_PATH
 
-# Check domain/path availability
+app=$YNH_APP_INSTANCE_NAME
-sudo yunohost app checkurl $domain$path -a ffsync
+final_path="/opt/yunohost/$app"
-if [[ ! $? -eq 0 ]]; then
-    ynh_die
-fi
 
-# Generate random password
+# Normalize the url path syntax
-db_pwd=$(head -c 8 /dev/urandom | sha1sum | cut -d " " -f1)
+path_url=$(ynh_normalize_url_path "$path_url")
 
-# Use 'FSyncMS' as database name and user
+#=================================================
-db_user=ffsync
+# STORE SETTINGS FROM MANIFEST
+#=================================================
 
-# Initialize database and store mysql password for upgrade
+ynh_app_setting_set "$app" final_path "$final_path"
-sudo yunohost app initdb $db_user -p $db_pwd
+
-ynh_app_setting_set ffsync mysqlpwd $db_pwd
+#=================================================
+# CHECK IF THE APP CAN BE INSTALLED WITH THESE ARGS
+#=================================================
+
+# Check destination directory
+test ! -e "$final_path" || ynh_die "This path already contains a folder"
+
+# Check web path availability
+ynh_webpath_available "$domain" "$path_url"
+# Register (book) web path
+ynh_webpath_register "$app" "$domain" "$path_url"
+
+#=================================================
+# STANDARD MODIFICATIONS
+#=================================================
 
-# Generate random password and save
+
-secret=$(head -c 20 /dev/urandom | sha1sum | cut -d " " -f1)
+#=================================================
-ynh_app_setting_set ffsync secret $secret
+# INSTALL DEPENDENCIES
+#=================================================
 
 # Check depends installation
-sudo apt-get install make python-dev python-virtualenv -y
+ynh_install_app_dependencies python-dev python-virtualenv \
+			 uwsgi uwsgi-plugin-python \
+			 `# ARM support: ` \
+			 build-essential libssl-dev libffi-dev
 
-# Check Swap
+#=================================================
-if [ $(sudo swapon -s | wc -l) = 1 ];
+# CREATE A MYSQL DATABASE
-then
+#=================================================
-	# It is NOT possible to setup a swap file on a tmpfs filesystem
-	mount | grep /tmp | grep tmpfs > /dev/null 2>&1
-	if [ $? = 1 ];
-	then
-		tmp_swap_file=/tmp/ffsync_swapfile
-	else
-		tmp_swap_file=/var/cache/ffsync_swapfile
-	fi
-	sudo dd if=/dev/zero of=$tmp_swap_file bs=1M count=256
-	sudo chmod 600 $tmp_swap_file
-	sudo mkswap $tmp_swap_file
-	sudo swapon $tmp_swap_file
-fi
 
+# Use 'FSyncMS' as database name and user
+db_user=$app
+db_name=$(ynh_sanitize_dbid $app)
 
-# Modify assets to take path into account
+ynh_app_setting_set "$app" db_name "$db_name"
-sudo find ../sources/syncserver/page/sync_files/ -type f -exec sed -i -e "s@media\/img@$path\/media\/img@g" {} \;
+ynh_mysql_setup_db "$db_user" "$db_name"
 
-# Copy files to the right place
+#=================================================
-final_path=/opt/yunohost/ffsync
+# DOWNLOAD, CHECK AND UNPACK SOURCE
-sudo mkdir -p $final_path
+#=================================================
-sudo cp -a ../sources/* $final_path
-sudo cp ../conf/ffsync /etc/init.d/
-sudo cp ../conf/ffsync.logrotate /etc/logrotate.d/ffsync
-sudo touch /var/log/ffsync.log
 
-# Set permissions to ffsync directory
+# Download, check integrity, uncompress and patch the source from app.src
-sudo useradd ffsync -d $final_path
+ynh_setup_source "$final_path"
-sudo chown ffsync:ffsync -R $final_path
-sudo chown ffsync /var/log/ffsync.log
 
-# Modify Nginx configuration file and copy it to Nginx conf directory
+# Modify assets to take path into account
-sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf
+# TODO: try to include this as a patch if still needed
-sed -i "s@ALIASTOCHANGE@$final_path/@g" ../conf/nginx.conf
+# find ../sources/syncserver/page/sync_files/ -type f -exec sed -i -e "s@media\/img@$path_url\/media\/img@g" {} \;
-sudo cp ../conf/nginx.conf /etc/nginx/conf.d/$domain.d/ffsync.conf
-sudo cp ../conf/syncserver.ini $final_path/syncserver.ini
-sudo sed -i -e "s@ynhbaseurl@$domain$path@g" $final_path/syncserver.ini
-sudo sed -i -e "s@changesecret@$secret@g" $final_path/syncserver.ini
-sudo sed -i "s/yunouser/$db_user/g" $final_path/syncserver.ini
-sudo sed -i "s/yunopass/$db_pwd/g" $final_path/syncserver.ini
-sudo sed -i "s/yunobase/$db_user/g" $final_path/syncserver.ini
-sudo sed -i -e "s@media\/img@$path\/media\/img@g" $final_path/syncserver/page/sync_files/firefox_sync-bundle.css
-sudo sed -i -e "s@media\/img@$path\/media\/img@g" $final_path/syncserver/page/sync_files/responsive-bundle.css
 
-# Init virtualenv
+#=================================================
-cd $final_path && sudo make build && sudo ./local/bin/easy_install gunicorn
+# NGINX CONFIGURATION
+#=================================================
 
-# Disable swapfile
+# Modify Nginx configuration file and copy it to Nginx conf directory
-if [ -z ${tmp_swap_file+x} ];
+ynh_add_nginx_config
+
+if [ "$path_url" == "/" ]
 then
-        sudo swapoff $tmp_swap_file
+	# $finalnginxconf comes from ynh_add_nginx_config
-        sudo rm -f $tmp_swap_file
+	# uwsgi_param is only needed for non-root installation
+	ynh_replace_string "uwsgi_param "     "#uwsgi_param "     "$finalnginxconf"
+	ynh_replace_string "uwsgi_modifier1 " "#uwsgi_modifier1 " "$finalnginxconf"
 fi
+ynh_store_file_checksum "$finalnginxconf"
+systemctl reload nginx
+
+#=================================================
+# CREATE DEDICATED USER
+#=================================================
+
+ynh_system_user_create "$app" "$final_path"
+
+#=================================================
+# SPECIFIC SETUP
+#=================================================
+# pip installation
+#=================================================
+
+virtualenv "$final_path/local"
+# Init virtualenv
+(
+	set +o nounset
+	source "$final_path/local/bin/activate"
+	set -o nounset
+	cd "$final_path"
+	pip install --upgrade pip
+	pip install pyramid_chameleon
+	CFLAGS="-Wno-error -Wno-error=format-security" \
+		ARCHFLAGS="-Wno-error=unused-command-line-argument-hard-error-in-future" \
+		pip install --requirement "$final_path/requirements.txt"
+
+	python "$final_path/setup.py" develop
+
+	touch "$final_path/local/COMPLETE"
+)
+
+# Add nice homepage
+cp -r ../sources/page $final_path/syncserver/
+(cd "$final_path/syncserver" && patch -p1 < $YNH_CWD/../sources/homepage.patch) || echo "Unable to apply patches"
+
+#=================================================
+# SECURE FILES AND DIRECTORIES
+#=================================================
+
+chown $app -R $final_path
+chmod u=rwX,g=rX,o= -R $final_path
+
+
+#=================================================
+# SETUP UWSGI
+#=================================================
+
+# Generate random password and save
+secret=$(ynh_string_random)
+ynh_app_setting_set "$app" secret "$secret"
+
+# create config file syncserver.ini
+rm "$final_path/syncserver.ini"
+ln -s "/etc/uwsgi/apps-available/$app.ini" "$final_path/syncserver.ini"
+
+# configure uwsgi
+ynh_add_uwsgi_service 'domain secret db_user db_pwd db_name'
+
+#=================================================
+# MODIFY A CONFIG FILE
+#=================================================
+
+# TODO: fix this css patch
+# ynh_replace_string "media\/img@$path_url\/media\/img@g" $final_path/syncserver/page/sync_files/firefox_sync-bundle.css
+# ynh_replace_string "media\/img@$path_url\/media\/img@g" $final_path/syncserver/page/sync_files/responsive-bundle.css
+
+#=================================================
+# GENERIC FINALIZATION
+#=================================================
+# SETUP SSOWAT
+#=================================================
 
-# Fix permission
+# accessible by everyone (authentification is done by firefox accounts)
-sudo find $final_path/ -type d -exec chmod 2755 {} \;
+ynh_app_setting_set "$app" skipped_uris "/"
-sudo find $final_path/ -type f -exec chmod g+r,o+r {} \;
-
-#enable services
-sudo chmod +x /etc/init.d/ffsync
-sudo update-rc.d ffsync defaults
-sudo service ffsync restart
-sudo service ffsync restart
-sudo service ffsync restart
-
-# Reload Nginx and regenerate SSOwat conf
-sudo yunohost app ssowatconf
-sudo service nginx restart
-sudo yunohost service add ffsync -l /var/log/ffsync.log
-ynh_app_setting_set ffsync skipped_uris "/"

scripts/remove

@@ -1,22 +1,63 @@
 #!/bin/bash
 
-# Source app helpers
+#=================================================
-. /usr/share/yunohost/helpers
+# GENERIC START
-
+#=================================================
-db_user=ffsync
+# IMPORT GENERIC HELPERS
-db_name=ffsync
+#=================================================
-root_pwd=$(sudo cat /etc/yunohost/mysql)
+
-domain=$(ynh_app_setting_get ffsync domain)
+source _common.sh
-
+source /usr/share/yunohost/helpers
-mysql -u root -p$root_pwd -e "DROP DATABASE $db_name ; DROP USER $db_user@localhost ;"
+
-sudo rm -rf /opt/yunohost/ffsync
+#=================================================
-sudo rm -f /etc/nginx/conf.d/$domain.d/ffsync.conf
+# LOAD SETTINGS
-sudo service ffsync stop
+#=================================================
-sudo update-rc.d ffsync remove
+
-sudo rm /etc/init.d/ffsync
+app=$YNH_APP_INSTANCE_NAME
-sudo rm /etc/logrotate.d/ffsync
+
-sudo yunohost service remove ffsync
+domain=$(ynh_app_setting_get "$app" domain)
-
+final_path=$(ynh_app_setting_get "$app" final_path)
-sudo service nginx reload
+db_user=$app
-sudo userdel ffsync
+db_name=$app
-sudo delgroup ffsync
+
+#=================================================
+# STANDARD REMOVE
+#=================================================
+# REMOVE UWSGI
+#=================================================
+
+ynh_remove_uwsgi_service
+
+#=================================================
+# REMOVE DEPENDENCIES
+#=================================================
+
+# Remove metapackage and its dependencies
+ynh_remove_app_dependencies
+
+#=================================================
+# REMOVE THE MYSQL DATABASE
+#=================================================
+
+ynh_mysql_remove_db "$db_user" "$db_name"
+
+#=================================================
+# REMOVE APP MAIN DIR
+#=================================================
+
+ynh_secure_remove "$final_path"
+
+#=================================================
+# REMOVE NGINX CONFIGURATION
+#=================================================
+
+ynh_remove_nginx_config
+
+#=================================================
+# GENERIC FINALIZATION
+#=================================================
+# REMOVE DEDICATED USER
+#=================================================
+
+# Delete a system user
+ynh_system_user_delete "$app"

scripts/restore

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+#=================================================
+# GENERIC START
+#=================================================
+# IMPORT GENERIC HELPERS
+#=================================================
+
+source ../settings/scripts/_common.sh
+source /usr/share/yunohost/helpers
+
+#=================================================
+# MANAGE SCRIPT FAILURE
+#=================================================
+
+# Exit if an error occurs during the execution of the script
+ynh_abort_if_errors
+
+#=================================================
+# LOAD SETTINGS
+#=================================================
+
+app=$YNH_APP_INSTANCE_NAME
+
+domain=$(ynh_app_setting_get "$app" domain)
+path_url=$(ynh_app_setting_get "$app" path)
+final_path=$(ynh_app_setting_get "$app" final_path)
+db_name=$(ynh_app_setting_get "$app" db_name)
+
+#=================================================
+# CHECK IF THE APP CAN BE RESTORED
+#=================================================
+
+ynh_webpath_available "$domain" "$path_url" \
+	|| ynh_die "Path not available: ${domain}${path_url}"
+test ! -d "$final_path" \
+	|| ynh_die "There is already a directory: $final_path "
+
+#=================================================
+# STANDARD RESTORATION STEPS
+#=================================================
+
+# Restore all config and data
+ynh_restore
+
+#=================================================
+# RESTORE THE MYSQL DATABASE
+#=================================================
+
+db_pwd=$(ynh_app_setting_get "$app" mysqlpwd)
+ynh_mysql_setup_db "$db_name" "$db_name" "$db_pwd"
+ynh_mysql_connect_as "$db_name" "$db_pwd" "$db_name" < ./db.sql
+
+#=================================================
+# RECREATE THE DEDICATED USER
+#=================================================
+
+# Create the dedicated user (if not existing)
+ynh_system_user_create "$app"
+
+#=================================================
+# RESTORE USER RIGHTS
+#=================================================
+
+chown $app -R $final_path
+chmod u=rwX,g=rX,o= -R $final_path
+
+#=================================================
+# SPECIFIC RESTORATION
+#=================================================
+# REINSTALL DEPENDENCIES
+#=================================================
+
+ynh_install_app_dependencies python-dev python-virtualenv \
+			 uwsgi uwsgi-plugin-python \
+			 `# ARM support: ` \
+			 build-essential libssl-dev libffi-dev
+
+# set authorizations
+chown $app:root /var/log/uwsgi/$app
+chmod -R u=rwX,g=rX,o= /var/log/uwsgi/$app
+
+#=================================================
+# RESTORE SERVICE
+#=================================================
+
+usermod --append --groups www-data "$app"
+
+systemctl daemon-reload
+systemctl enable "uwsgi-app@$app.service"
+
+#=================================================
+# ADVERTISE SERVICE IN ADMIN PANEL
+#=================================================
+
+yunohost service add "uwsgi-app@$app.service" --log "/var/log/uwsgi/app/$app"
+
+#=================================================
+# GENERIC FINALIZATION
+#=================================================
+# RELOAD NGINX AND PHP-FPM
+#=================================================
+
+systemctl start "uwsgi-app@$app.service"
+systemctl reload nginx

scripts/upgrade

@@ -1,91 +1,182 @@
 #!/bin/bash
 
-# Source app helpers
+#=================================================
-. /usr/share/yunohost/helpers
+# GENERIC START
+#=================================================
+# IMPORT GENERIC HELPERS
+#=================================================
+
+source _common.sh
+source /usr/share/yunohost/helpers
+
+#=================================================
+# MANAGE SCRIPT FAILURE
+#=================================================
+
+# Exit if an error occurs during the execution of the script
+ynh_abort_if_errors
+
+#=================================================
+# RETRIEVE ARGUMENTS FROM THE MANIFEST
+#=================================================
+
+app=$YNH_APP_INSTANCE_NAME
 
 # Retrieve arguments
-domain=$(ynh_app_setting_get ffsync domain)
+domain=$(ynh_app_setting_get "$app" domain)
-path=$(ynh_app_setting_get ffsync path)
+path_url=$(ynh_app_setting_get "$app" path)
-db_pwd=$(ynh_app_setting_get ffsync mysqlpwd)
+db_name=$(ynh_app_setting_get "$app" db_name)
-db_user=ffsync
+db_pwd=$(ynh_app_setting_get "$app" mysqlpwd)
-final_path=/opt/yunohost/ffsync
+db_user=$app
-
+final_path=$(ynh_app_setting_get "$app" final_path)
-# Get secret variable
+secret=$(ynh_app_setting_get "$app" secret)
-secret=$(ynh_app_setting_get ffsync secret)
+
-# Get from conf file if not defined
+#=================================================
-if [[ -z $secret ]]
+# ENSURE DOWNWARD COMPATIBILITY
-then
+#=================================================
-	secret=$(sudo grep "secret =" $final_path/syncserver.ini | cut -d" " -f3)
+
-	ynh_app_setting_set ffsync secret $secret
+# If db_name doesn't exist, create it
+if [ -z "$db_name" ]; then
+	db_name=$(ynh_sanitize_dbid "$app")
+	ynh_app_setting_set $app db_name "$db_name"
 fi
 
-# Check Swap
+# If final_path doesn't exist, create it
-if [ $(sudo swapon -s | wc -l) = 1 ];
+if [ -z "$final_path" ]; then
-then
+	final_path=/opt/yunohost/$app
-	# It is NOT possible to setup a swap file on a tmpfs filesystem
+	ynh_app_setting_set "$app" final_path "$final_path"
-	mount | grep /tmp | grep tmpfs > /dev/null 2>&1
-	if [ $? = 1 ];
-	then
-		tmp_swap_file=/tmp/ffsync_swapfile
-	else
-		tmp_swap_file=/var/cache/ffsync_swapfile
 fi
-	sudo dd if=/dev/zero of=$tmp_swap_file bs=1M count=256
+
-	sudo chmod 600 $tmp_swap_file
+# If path_url doesn't exist, create it
-	sudo mkswap $tmp_swap_file
+if [ -z "$path_url" ]; then
-	sudo swapon $tmp_swap_file
+	path_url=$(ynh_app_setting_get "$app" path)
+	ynh_app_setting_set "$app" path "$path_url"
+	ynh_app_setting_delete "$app" path
 fi
 
-# Copy files to the right place
+# Detect old installation style
-sudo mkdir -p $final_path
+if [ -e /etc/init.d/ffsync ]; then
-sudo cp -a ../sources/* $final_path
+	service ffsync stop
-sudo cp ../conf/ffsync /etc/init.d/
+	update-rc.d -f ffsync remove
-sudo cp ../conf/ffsync.logrotate /etc/logrotate.d/ffsync
-
-
-# Set permissions to ffsync directory
-sudo useradd ffsync -d $final_path
-sudo chown ffsync:ffsync -R $final_path
-
-# Modify Nginx configuration file and copy it to Nginx conf directory
-sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf
-sed -i "s@ALIASTOCHANGE@$final_path@g" ../conf/nginx.conf
-sudo cp ../conf/nginx.conf /etc/nginx/conf.d/$domain.d/ffsync.conf
-sudo cp ../conf/syncserver.ini $final_path/syncserver.ini
-sudo sed -i -e "s@ynhbaseurl@$domain$path@g" $final_path/syncserver.ini
-sudo sed -i -e "s@changesecret@$secret@g" $final_path/syncserver.ini
-sudo sed -i "s/yunouser/$db_user/g" $final_path/syncserver.ini
-sudo sed -i "s/yunopass/$db_pwd/g" $final_path/syncserver.ini
-sudo sed -i "s/yunobase/$db_user/g" $final_path/syncserver.ini
-sudo sed -i -e "s@media\/img@$path\/media\/img@g" $final_path/syncserver/page/sync_files/firefox_sync-bundle.css
-sudo sed -i -e "s@media\/img@$path\/media\/img@g" $final_path/syncserver/page/sync_files/responsive-bundle.css
-
-# stop service before upgrade
-sudo service ffsync stop
 
-# Init virtualenv
+	ynh_secure_remove /etc/init.d/ffsync
-cd $final_path && sudo make build && sudo ./local/bin/easy_install gunicorn
+	ynh_secure_remove /var/log/ffsync.log
+	ynh_secure_remove /opt/yunohost/ffsync
+
+	yunohost service remove "$app"
+fi
+
+#=================================================
+# BACKUP BEFORE UPGRADE THEN ACTIVE TRAP
+#=================================================
+
+# Backup the current version of the app
+ynh_backup_before_upgrade
+ynh_clean_setup () {
+	# restore it if the upgrade fails
+	ynh_restore_upgradebackup
+}
+# Exit if an error occurs during the execution of the script
+ynh_abort_if_errors
+
+#=================================================
+# CHECK THE PATH
+#=================================================
+
+# Normalize the URL path syntax
+path_url=$(ynh_normalize_url_path "$path_url")
+
+#=================================================
+# STANDARD UPGRADE STEPS
+#=================================================
+# INSTALL DEPENDENCIES
+#=================================================
 
-# Disable swapfile
+# Check depends installation
-if [ -z ${tmp_swap_file+x} ];
+ynh_install_app_dependencies python-dev python-virtualenv \
+			 uwsgi uwsgi-plugin-python \
+			 `# ARM support: ` \
+			 build-essential libssl-dev libffi-dev
+
+#=================================================
+# DOWNLOAD, CHECK AND UNPACK SOURCE
+#=================================================
+
+# Download, check integrity, uncompress and patch the source from app.src
+ynh_setup_source "$final_path"
+
+#=================================================
+# NGINX CONFIGURATION
+#=================================================
+
+# Create a dedicated nginx config
+ynh_add_nginx_config
+
+if [ "$path_url" == "/" ]
 then
-        sudo swapoff $tmp_swap_file
+	# $finalnginxconf comes from ynh_add_nginx_config
-        sudo rm -f $tmp_swap_file
+	# uwsgi_param is only needed for non-root installation
+	ynh_replace_string "uwsgi_param "     "#uwsgi_param "     "$finalnginxconf"
+	ynh_replace_string "uwsgi_modifier1 " "#uwsgi_modifier1 " "$finalnginxconf"
 fi
+ynh_store_file_checksum "$finalnginxconf"
+systemctl reload nginx
+
+#=================================================
+# CREATE DEDICATED USER
+#=================================================
+
+# Create a system user
+ynh_system_user_create "$app"
+
+#=================================================
+# pip installation
+#=================================================
+
+virtualenv "$final_path/local"
+# Init virtualenv
+(
+	set +o nounset
+	source "$final_path/local/bin/activate"
+	set -o nounset
+	cd "$final_path"
+	pip install --upgrade pip
+	pip install pyramid_chameleon
+	CFLAGS="-Wno-error -Wno-error=format-security" \
+		ARCHFLAGS="-Wno-error=unused-command-line-argument-hard-error-in-future" \
+		pip install --requirement "$final_path/requirements.txt"
+
+	python "$final_path/setup.py" develop
+
+	touch "$final_path/local/COMPLETE"
+)
+
+# Add nice homepage
+cp -r ../sources/page $final_path/syncserver/
+(cd "$final_path/syncserver" && patch -p1 < $YNH_CWD/../sources/homepage.patch) || echo "Unable to apply patches"
+
+#=================================================
+# SECURE FILES AND DIRECTORIES
+#=================================================
+
+chown $app -R $final_path
+chmod u=rwX,g=rX,o= -R $final_path
+
+#=================================================
+# SETUP UWSGI
+#=================================================
+
+# create config file syncserver.ini
+rm "$final_path/syncserver.ini"
+ln -s "/etc/uwsgi/apps-available/$app.ini" "$final_path/syncserver.ini"
+
+# configure uwsgi
+ynh_add_uwsgi_service 'domain secret db_user db_pwd db_name'
+
+#=================================================
+# GENERIC FINALIZATION
+#=================================================
+# SETUP SSOWAT
+#=================================================
 
-# Fix permission
+ynh_app_setting_set "$app" skipped_uris "/"
-sudo find $final_path/ -type d -exec chmod 2755 {} \;
-sudo find $final_path/ -type f -exec chmod g+r,o+r {} \;
-sudo usermod -a -G ffsync www-data
-
-#enable services
-sudo chmod +x /etc/init.d/ffsync
-sudo update-rc.d ffsync defaults
-sudo service ffsync restart
-sudo service ffsync restart
-sudo service ffsync restart
-
-# Reload Nginx and regenerate SSOwat conf
-sudo service nginx reload
-ynh_app_setting_set ffsync skipped_uris "/"
-sudo yunohost app ssowatconf

sources/Dockerfile

@@ -1,48 +0,0 @@
-##########################################################
-#                  /!\ WARNING /!\                       #
-# This is completely experimental. Use at your own risk. #
-#             Also, learn you some docker:               #
-#           http://docker.io/gettingstarted              #
-##########################################################
-
-FROM debian:7.4
-MAINTAINER Dan Callahan <dan.callahan@gmail.com>
-
-# Base system setup
-
-RUN DEBIAN_FRONTEND=noninteractive apt-get update \
-    && apt-get install --no-install-recommends -y \
-    vim locales \
-    && apt-get clean
-
-RUN locale-gen C.UTF-8 && LANG=C.UTF-8 /usr/sbin/update-locale
-
-ENV LANG C.UTF-8
-
-RUN useradd --create-home app
-
-# Build the Sync server
-
-RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-    ca-certificates \
-    build-essential \
-    libzmq-dev \
-    python-dev \
-    python-virtualenv \
-    && apt-get clean
-
-USER app
-
-RUN mkdir -p /home/app/syncserver
-ADD Makefile *.ini *.wsgi *.rst *.txt *.py /home/app/syncserver/
-ADD ./syncserver/ /home/app/syncserver/syncserver/
-WORKDIR /home/app/syncserver
-
-RUN make build
-
-# Run the Sync server
-
-EXPOSE 5000
-
-ENTRYPOINT ["/usr/bin/make"]
-CMD ["serve"]

sources/MANIFEST.in

@@ -1,3 +0,0 @@
-include syncserver.ini
-include syncserver.wsgi
-include syncserver/tests.ini

sources/Makefile

@@ -1,50 +0,0 @@
-SYSTEMPYTHON = `which python2 python | head -n 1`
-VIRTUALENV = virtualenv --python=$(SYSTEMPYTHON)
-ENV = ./local
-TOOLS := $(addprefix $(ENV)/bin/,flake8 nosetests)
-
-# Hackety-hack around OSX system python bustage.
-# The need for this should go away with a future osx/xcode update.
-ARCHFLAGS = -Wno-error=unused-command-line-argument-hard-error-in-future
-
-# Hackety-hack around errors duing compile of ultramemcached.
-CFLAGS = "-Wno-error -Wno-error=format-security"
-
-INSTALL = CFLAGS=$(CFLAGS) ARCHFLAGS=$(ARCHFLAGS) $(ENV)/bin/pip install
-
-
-.PHONY: all
-all: build
-
-.PHONY: build
-build: | $(ENV)/COMPLETE
-$(ENV)/COMPLETE: requirements.txt
-	$(VIRTUALENV) --no-site-packages $(ENV)
-	$(INSTALL) -r requirements.txt
-	$(ENV)/bin/python ./setup.py develop
-	touch $(ENV)/COMPLETE
-
-.PHONY: test
-test: | $(TOOLS)
-	$(ENV)/bin/flake8 ./syncserver
-	$(ENV)/bin/nosetests -s syncstorage.tests
-	# Tokenserver tests currently broken due to incorrect file paths
-	# $(ENV)/bin/nosetests -s tokenserver.tests
-	
-	# Test against a running server
-	$(ENV)/bin/gunicorn --paste syncserver/tests.ini 2> /dev/null & SERVER_PID=$$!; \
-	sleep 2; \
-	$(ENV)/bin/python -m syncstorage.tests.functional.test_storage \
-		--use-token-server http://localhost:5000/token/1.0/sync/1.5; \
-	kill $$SERVER_PID
-
-$(TOOLS): | $(ENV)/COMPLETE
-	$(INSTALL) nose flake8
-
-.PHONY: serve
-serve: | $(ENV)/COMPLETE
-	$(ENV)/bin/gunicorn --paste ./syncserver.ini
-
-.PHONY: clean
-clean:
-	rm -rf $(ENV)

sources/README.rst

@@ -1,73 +0,0 @@
-Run-Your-Own Firefox Sync Server
-================================
-
-This is an all-in-one package for running a self-hosted Firefox Sync server.
-If bundles the "tokenserver" project for authentication and the "syncstorage"
-project for storage, produce a single stand-alone webapp.
-
-Complete installation instructions are available at:
-
-   https://docs.services.mozilla.com/howtos/run-sync-1.5.html
-
-
-Quickstart
-----------
-
-The Sync Server software runs using **python 2.6** or later, and the build
-process requires **make** and **virtualenv**.  You will need to have the
-following packages (or similar, depending on your operating system) installed:
-
-- python2.7
-- python2.7-dev
-- python-virtualenv
-- make
-
-Take a checkout of this repository, then run "make build" to pull in the 
-necessary python package dependencies::
-
-    $ git clone https://github.com/mozilla-services/syncserver
-    $ cd syncserver
-    $ make build
-
-To sanity-check that things got installed correctly, do the following::
-
-    $ make test
-
-Now you can run the server::
-
-    $ make serve
-
-This should start a server on http://localhost:5000/.  
-
-Now go into Firefox's `about:config` page, search for a setting named
-"tokenServerURI", and change it to point to your server::
-
-    services.sync.tokenServerURI:  http://localhost:5000/token/1.0/sync/1.5
-
-Firefox should now sync against your local server rather than the default
-Mozilla-hosted servers.
-
-For more details on setting up a stable deployment, see:
-
-   https://docs.services.mozilla.com/howtos/run-sync-1.5.html
-
-
-Customization
--------------
-
-All customization of the server can be done by editing the file
-"syncserver.ini", which contains lots of comments to help you on
-your way.  Things you might like to change include:
-
-    * The client-visible hostname for your server.  Edit the "public_url"
-      key under the [syncstorage] section.
-
-    * The database in which to store sync data.  Edit the "sqluri" setting
-      under the [syncstorage] section.
-
-
-Questions, Feedback
--------------------
-
-- IRC channel: #sync. See http://irc.mozilla.org/
-- Mailing list: https://mail.mozilla.org/listinfo/services-dev

sources/homepage.patch

@@ -0,0 +1,48 @@
+diff -Naur a/__init__.py b/__init__.py
+--- a/__init__.py 2018-08-22 17:54:00.369070610 +0200
++++ b/__init__.py 2018-08-22 20:11:39.861463512 +0200
+@@ -11,6 +11,10 @@
+ from pyramid.response import Response
+ from pyramid.events import NewRequest, subscriber
+ 
++from pyramid.static import static_view
++from pyramid.view import view_config
++from pyramid.renderers import render, render_to_response
++
+ try:
+     import requests.packages.urllib3.contrib.pyopenssl
+     HAS_PYOPENSSL = True
+@@ -135,13 +139,27 @@
+     config.scan("syncserver", ignore=["syncserver.wsgi_app"])
+     config.include("syncstorage", route_prefix="/storage")
+     config.include("tokenserver", route_prefix="/token")
++    config.include('pyramid_chameleon')
+ 
+-    # Add a top-level "it works!" view.
+-    def itworks(request):
+-        return Response("it works!")
+-
+-    config.add_route('itworks', '/')
+-    config.add_view(itworks, route_name='itworks')
++    # Add a top-level explaination view.
++    # First view, available at http://localhost:6543/
++    def page(request):
++               result = render('page/index.pt',
++                               {'public_url':public_url},
++                               request=request)
++               response = Response(result)
++               return response
++    config.add_route('page', '/')
++    config.add_view(page, route_name='page')
++
++    www = static_view(
++        os.path.realpath(os.path.dirname(__file__)+"/page/"),
++        use_subpath=True
++    )
++    # Documentation for Hybrid routing can be found here
++    # http://docs.pylonsproject.org/projects/pyramid/en/1.0-branch/narr/hybrid.html#using-subpath-in-a-route-pattern
++    config.add_route('index', '/*subpath', 'www')  # subpath is a reserved word
++    config.add_view(www, route_name='index')
+ 
+ 
+ def import_settings_from_environment_variables(settings, environ=None):

sources/requirements.txt

@@ -1,13 +0,0 @@
-cornice==0.16.2
-gunicorn==19.1.1
-pyramid==1.5
-pyramid_chameleon==0.3
-requests==2.7
-simplejson==3.4
-SQLAlchemy==0.9.4
-unittest2==0.5.1
-zope.component==4.2.1
-configparser==3.5.0b2
-https://github.com/mozilla-services/mozservices/archive/e00e1b68130423ad98d0f6185655bde650443da8.zip
-https://github.com/mozilla-services/tokenserver/archive/d7e513e8a4f5c588b70d685a8df1d2e508c341c0.zip
-http://github.com/mozilla-services/server-syncstorage/archive/1.5.5.zip

sources/setup.py

@@ -1,16 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-from setuptools import setup
-
-entry_points = """
-[paste.app_factory]
-main = syncserver:main
-"""
-
-setup(
-    name='syncserver',
-    version="1.5.2",
-    packages=['syncserver'],
-    entry_points=entry_points
-)

sources/syncserver.wsgi

@@ -1,40 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import os
-import sys
-import site
-from logging.config import fileConfig
-from ConfigParser import NoSectionError
-
-# detecting if virtualenv was used in this dir
-_CURDIR = os.path.dirname(os.path.abspath(__file__))
-_PY_VER = sys.version.split()[0][:3]
-_SITE_PKG = os.path.join(_CURDIR, 'local', 'lib', 'python' + _PY_VER, 'site-packages')
-
-# adding virtualenv's site-package and ordering paths
-saved = sys.path[:]
-
-if os.path.exists(_SITE_PKG):
-    site.addsitedir(_SITE_PKG)
-
-for path in sys.path:
-    if path not in saved:
-        saved.insert(0, path)
-
-sys.path[:] = saved
-
-# setting up the egg cache to a place where apache can write
-os.environ['PYTHON_EGG_CACHE'] = '/tmp/python-eggs'
-
-# setting up logging
-ini_file = os.path.join(_CURDIR, 'syncserver.ini')
-try:
-    fileConfig(ini_file)
-except NoSectionError:
-    pass
-
-# running the app using Paste
-from paste.deploy import loadapp
-application = loadapp('config:%s'% ini_file)

sources/syncserver/__init__.py

@@ -1,174 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import os
-import logging
-from urlparse import urlparse, urlunparse
-
-from pyramid.events import NewRequest, subscriber
-from pyramid.static import static_view
-from pyramid.view import view_config
-from pyramid.renderers import render, render_to_response
-from pyramid.response import Response
-
-import mozsvc.config
-
-from tokenserver.util import _JSONError
-
-logger = logging.getLogger("syncserver")
-
-
-def includeme(config):
-    """Install SyncServer application into the given Pyramid configurator."""
-    # Set the umask so that files are created with secure permissions.
-    # Necessary for e.g. created-on-demand sqlite database files.
-    os.umask(0077)
-
-    # Sanity-check the deployment settings and provide sensible defaults.
-    settings = config.registry.settings
-    public_url = settings.get("syncserver.public_url")
-    if public_url is None:
-        raise RuntimeError("you much configure syncserver.public_url")
-    public_url = public_url.rstrip("/")
-    settings["syncserver.public_url"] = public_url
-
-    secret = settings.get("syncserver.secret")
-    if secret is None:
-        secret = os.urandom(32).encode("hex")
-    sqluri = settings.get("syncserver.sqluri")
-    if sqluri is None:
-        rootdir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
-        sqluri = "sqlite:///" + os.path.join(rootdir, "syncserver.db")
-
-    # Configure app-specific defaults based on top-level configuration.
-    settings.pop("config", None)
-    if "tokenserver.backend" not in settings:
-        # Default to our simple static node-assignment backend
-        settings["tokenserver.backend"] =\
-            "syncserver.staticnode.StaticNodeAssignment"
-        settings["tokenserver.sqluri"] = sqluri
-        settings["tokenserver.node_url"] = public_url
-        settings["endpoints.sync-1.5"] = "{node}/storage/1.5/{uid}"
-    if "tokenserver.monkey_patch_gevent" not in settings:
-        # Default to no gevent monkey-patching
-        settings["tokenserver.monkey_patch_gevent"] = False
-    if "tokenserver.applications" not in settings:
-        # Default to just the sync-1.5 application
-        settings["tokenserver.applications"] = "sync-1.5"
-    if "tokenserver.secrets.backend" not in settings:
-        # Default to a single fixed signing secret
-        settings["tokenserver.secrets.backend"] = "mozsvc.secrets.FixedSecrets"
-        settings["tokenserver.secrets.secrets"] = [secret]
-    if "tokenserver.allow_new_users" not in settings:
-        allow_new_users = settings.get("syncserver.allow_new_users")
-        if allow_new_users is not None:
-            settings["tokenserver.allow_new_users"] = allow_new_users
-    if "hawkauth.secrets.backend" not in settings:
-        # Default to the same secrets backend as the tokenserver
-        for key in settings.keys():
-            if key.startswith("tokenserver.secrets."):
-                newkey = "hawkauth" + key[len("tokenserver"):]
-                settings[newkey] = settings[key]
-    if "storage.backend" not in settings:
-        # Default to sql syncstorage backend
-        settings["storage.backend"] = "syncstorage.storage.sql.SQLStorage"
-        settings["storage.sqluri"] = sqluri
-        settings["storage.create_tables"] = True
-    if "browserid.backend" not in settings:
-        # Default to remote verifier, with base of public_url as only audience
-        audience = urlunparse(urlparse(public_url)._replace(path=""))
-        settings["browserid.backend"] = "tokenserver.verifiers.RemoteVerifier"
-        settings["browserid.audiences"] = audience
-    if "loggers" not in settings:
-        # Default to basic logging config.
-        root_logger = logging.getLogger("")
-        if not root_logger.handlers:
-            logging.basicConfig(level=logging.INFO)
-
-    # Include the relevant sub-packages.
-    config.scan("syncserver")
-    config.include("syncstorage", route_prefix="/storage")
-    config.include("tokenserver", route_prefix="/token")
-    config.include('pyramid_chameleon')
-
-    # Add a top-level explaination view.
-    # First view, available at http://localhost:6543/
-    def page(request):
-		result = render('page/index.pt',
-		                {'public_url':public_url},
-		                request=request)
-		response = Response(result)
-		return response
-    config.add_route('page', '/')
-    config.add_view(page, route_name='page')
-    
-    www = static_view(
-        os.path.realpath(os.path.dirname(__file__)+"/page/"),
-        use_subpath=True
-    )
-    # Documentation for Hybrid routing can be found here
-    # http://docs.pylonsproject.org/projects/pyramid/en/1.0-branch/narr/hybrid.html#using-subpath-in-a-route-pattern
-    config.add_route('index', '/*subpath', 'www')  # subpath is a reserved word
-    config.add_view(www, route_name='index')
-
-
-@subscriber(NewRequest)
-def reconcile_wsgi_environ_with_public_url(event):
-    """Event-listener that checks and tweaks WSGI environ based on public_url.
-
-    This is a simple trick to help ensure that the configured public_url
-    matches the actual deployed address.  It fixes fixes parts of the WSGI
-    environ where it makes sense (e.g. SCRIPT_NAME) and warns about any parts
-    that seem obviously mis-configured (e.g. http:// versus https://).
-
-    It's very important to get public_url and WSGI environ matching exactly,
-    since they're used for browserid audience checking and HAWK signature
-    validation, so mismatches can easily cause strange and cryptic errors.
-    """
-    request = event.request
-    public_url = request.registry.settings["syncserver.public_url"]
-    p_public_url = urlparse(public_url)
-    # If we don't have a SCRIPT_NAME, take it from the public_url.
-    # This is often the case if we're behind e.g. an nginx proxy that
-    # is serving us at some sub-path.
-    if not request.script_name:
-        request.script_name = p_public_url.path.rstrip("/")
-    # If the environ does not match public_url, requests are almost certainly
-    # going to fail due to auth errors.  We can either bail out early, or we
-    # can forcibly clobber the WSGI environ with the values from public_url.
-    # This is a security risk if you've e.g. mis-configured the server, so
-    # it's not enabled by default.
-    application_url = request.application_url
-    if public_url != application_url:
-        if not request.registry.settings.get("syncserver.force_wsgi_environ"):
-            msg = "\n".join((
-                "The public_url setting doesn't match the application url.",
-                "This will almost certainly cause authentication failures!",
-                "    public_url setting is: %s" % (public_url,),
-                "    application url is:    %s" % (application_url,),
-                "You can disable this check by setting the force_wsgi_environ",
-                "option in your config file, but do so at your own risk.",
-            ))
-            logger.error(msg)
-            raise _JSONError([msg], status_code=500)
-        request.scheme = p_public_url.scheme
-        request.host = p_public_url.netloc
-        request.script_name = p_public_url.path.rstrip("/")
-
-
-def get_configurator(global_config, **settings):
-    """Load a SyncStorge configurator object from deployment settings."""
-    config = mozsvc.config.get_configurator(global_config, **settings)
-    config.begin()
-    try:
-        config.include(includeme)
-    finally:
-        config.end()
-    return config
-
-
-def main(global_config, **settings):
-    """Load a SyncStorage WSGI app from deployment settings."""
-    config = get_configurator(global_config, **settings)
-    return config.make_wsgi_app()

sources/syncserver/staticnode.py

@@ -1,221 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-"""
-Simple node-assignment backend using a single, static node.
-
-This is a greatly-simplified node-assignment backend.  It keeps user records
-in an SQL database, but does not attempt to do any node management.  All users
-are implicitly assigned to a single, static node.
-
-XXX TODO: move this into the tokenserver repo.
-
-"""
-import time
-import urlparse
-from mozsvc.exceptions import BackendError
-
-from sqlalchemy import Column, Integer, String, BigInteger, Index
-from sqlalchemy import create_engine, Table, MetaData
-from sqlalchemy.pool import QueuePool
-from sqlalchemy.sql import text as sqltext
-from sqlalchemy.exc import IntegrityError
-
-from tokenserver.assignment import INodeAssignment
-from zope.interface import implements
-
-
-metadata = MetaData()
-
-
-users = Table(
-    "users",
-    metadata,
-    Column("uid", Integer(), primary_key=True, autoincrement=True,
-           nullable=False),
-    Column("service", String(32), nullable=False),
-    Column("email", String(255), nullable=False),
-    Column("generation", BigInteger(), nullable=False),
-    Column("client_state", String(32), nullable=False),
-    Column("created_at", BigInteger(), nullable=False),
-    Column("replaced_at", BigInteger(), nullable=True),
-    Index('lookup_idx', 'email', 'service', 'created_at'),
-    Index('clientstate_idx', 'email', 'service', 'client_state', unique=True),
-)
-
-
-_GET_USER_RECORDS = sqltext("""\
-select
-    uid, generation, client_state
-from
-    users
-where
-    email = :email
-and
-    service = :service
-order by
-    created_at desc, uid desc
-limit
-    20
-""")
-
-
-_CREATE_USER_RECORD = sqltext("""\
-insert into
-    users
-    (service, email, generation, client_state, created_at, replaced_at)
-values
-    (:service, :email, :generation, :client_state, :timestamp, NULL)
-""")
-
-
-_UPDATE_GENERATION_NUMBER = sqltext("""\
-update
-    users
-set
-    generation = :generation
-where
-    service = :service and email = :email and
-    generation < :generation and replaced_at is null
-""")
-
-
-_REPLACE_USER_RECORDS = sqltext("""\
-update
-    users
-set
-    replaced_at = :timestamp
-where
-    service = :service and email = :email
-    and replaced_at is null and created_at < :timestamp
-""")
-
-
-def get_timestamp():
-    """Get current timestamp in milliseconds."""
-    return int(time.time() * 1000)
-
-
-class StaticNodeAssignment(object):
-    implements(INodeAssignment)
-
-    def __init__(self, sqluri, node_url, **kw):
-        self.sqluri = sqluri
-        self.node_url = node_url
-        self.driver = urlparse.urlparse(sqluri).scheme.lower()
-        sqlkw = {
-            "logging_name": "syncserver",
-            "connect_args": {},
-            "poolclass": QueuePool,
-            "pool_reset_on_return": True,
-        }
-        if self.driver == "sqlite":
-            # We must mark it as safe to share sqlite connections between
-            # threads.  The pool will ensure there's on race conditions.
-            sqlkw["connect_args"]["check_same_thread"] = False
-            # If using a :memory: database, we must use a QueuePool of size
-            # 1 so that a single connection is shared by all threads.
-            if urlparse.urlparse(sqluri).path.lower() in ("/", "/:memory:"):
-                sqlkw["pool_size"] = 1
-                sqlkw["max_overflow"] = 0
-        if "mysql" in self.driver:
-            # Guard against the db closing idle conections.
-            sqlkw["pool_recycle"] = 3600
-        self._engine = create_engine(sqluri, **sqlkw)
-        users.create(self._engine, checkfirst=True)
-
-    def get_user(self, service, email):
-        params = {'service': service, 'email': email}
-        res = self._engine.execute(_GET_USER_RECORDS, **params)
-        try:
-            row = res.fetchone()
-            if row is None:
-                return None
-            # The first row is the most up-to-date user record.
-            user = {
-                'email': email,
-                'uid': row.uid,
-                'node': self.node_url,
-                'generation': row.generation,
-                'client_state': row.client_state,
-                'old_client_states': {}
-            }
-            # Any subsequent rows are due to old client-state values.
-            row = res.fetchone()
-            while row is not None:
-                user['old_client_states'][row.client_state] = True
-                row = res.fetchone()
-            return user
-        finally:
-            res.close()
-
-    def allocate_user(self, service, email, generation=0, client_state=''):
-        params = {
-            'service': service, 'email': email, 'generation': generation,
-            'client_state': client_state, 'timestamp': get_timestamp()
-        }
-        try:
-            res = self._engine.execute(_CREATE_USER_RECORD, **params)
-        except IntegrityError:
-            raise
-            return self.get_user(service, email)
-        else:
-            res.close()
-            return {
-                'email': email,
-                'uid': res.lastrowid,
-                'node': self.node_url,
-                'generation': generation,
-                'client_state': client_state,
-                'old_client_states': {}
-            }
-
-    def update_user(self, service, user, generation=None, client_state=None):
-        if client_state is None:
-            # uid can stay the same, just update the generation number.
-            if generation is not None:
-                params = {
-                    'service': service,
-                    'email': user['email'],
-                    'generation': generation,
-                }
-                res = self._engine.execute(_UPDATE_GENERATION_NUMBER, **params)
-                res.close()
-                user['generation'] = max(generation, user['generation'])
-        else:
-            # reject previously-seen client-state strings.
-            if client_state == user['client_state']:
-                raise BackendError('previously seen client-state string')
-            if client_state in user['old_client_states']:
-                raise BackendError('previously seen client-state string')
-            # need to create a new record for new client_state.
-            if generation is not None:
-                generation = max(user['generation'], generation)
-            else:
-                generation = user['generation']
-            now = get_timestamp()
-            params = {
-                'service': service, 'email': user['email'],
-                'generation': generation, 'client_state': client_state,
-                'timestamp': now,
-            }
-            try:
-                res = self._engine.execute(_CREATE_USER_RECORD, **params)
-            except IntegrityError:
-                user.update(self.get_user(service, user['email']))
-            else:
-                self.get_user(service, user['email'])
-                user['uid'] = res.lastrowid
-                user['generation'] = generation
-                user['old_client_states'][user['client_state']] = True
-                user['client_state'] = client_state
-                res.close()
-            # mark old records as having been replaced.
-            # if we crash here, they are unmarked and we may fail to
-            # garbage collect them for a while, but the active state
-            # will be undamaged.
-            params = {
-                'service': service, 'email': user['email'], 'timestamp': now
-            }
-            res = self._engine.execute(_REPLACE_USER_RECORDS, **params)
-            res.close()

sources/syncserver/tests.ini

@@ -1,19 +0,0 @@
-[server:main]
-use = egg:gunicorn
-host = 0.0.0.0
-port = 5000
-workers = 1
-timeout = 30
-
-[app:main]
-use = egg:SyncServer
-
-[syncserver]
-# This must be edited to point to the public URL of your server.
-public_url = http://localhost:5000/
-
-# This defines the database in which to store all server data.
-#sqluri = sqlite:////tmp/syncserver.db
-
-# This is a secret key used for signing authentication tokens.
-#secret = INSERT_SECRET_KEY_HERE