firefox sync with gunicorn

conf/ffsync

@@ -0,0 +1,64 @@
+#!/bin/bash -x
+# /etc/init.d/sync
+# version 0.1 2013-03-12 (YYYY-MM-DD)
+
+### BEGIN INIT INFO
+# Provides:   sync
+# Required-Start: $local_fs $remote_fs
+# Required-Stop:  $local_fs $remote_fs
+# Should-Start:   $network
+# Should-Stop:    $network
+# Default-Start:  2 3 4 5
+# Default-Stop:   0 1 6
+# Short-Description:    Mozilla Sync server
+# Description:    Starts the mozilla sync server
+### END INIT INFO
+
+# Source function library.
+#. /etc/rc.d/init.d/functions
+
+prog=sync
+SYNC_USER=ffsync
+SYNC_HOME=/opt/yunohost/ffsync
+CPU_COUNT=2
+pidfile=/tmp/sync.pid
+lockfile=/var/run/mozilla/sync.lock
+conffile=${SYNC_HOME}/syncserver.ini
+GUNICORN=${SYNC_HOME}/local/bin/gunicorn_paster
+GUNICORN_ARGS="--access-logfile /var/log/ffsync.log --daemon -p $pidfile"
+#INVOCATION="$GUNICORN $GUNICORN_ARGS $conffile"
+
+start () {
+    echo -n "Starting $prog"
+    start-stop-daemon --start -c ${SYNC_USER} --exec $GUNICORN -- $GUNICORN_ARGS $conffile 
+    RETVAL=$?
+    echo
+    [ $RETVAL = 0 ] && touch ${lockfile}
+    return $RETVAL
+}
+
+stop() {
+    echo "Stopping $prog"
+    start-stop-daemon --stop --quiet --oknodo --pidfile ${pidfile}
+    #log_end_msg $?
+    rm -f ${pidfile}
+}
+
+case "$1" in
+    start)
+        start
+        ;;
+    stop)
+        stop
+        ;;
+    restart)
+        stop
+        start
+        ;;
+    *)
+        echo $"Usage: $prog {start|stop|restart|help}"
+        RETVAL=2
+esac
+
+exit $RETVAL
+

conf/nginx.conf

@@ -2,13 +2,14 @@ location PATHTOCHANGE {
         if ($scheme = http) {
                 rewrite ^ https://$server_name$request_uri? permanent;
         }
-        try_files $uri @ffsync;
+        proxy_set_header Host $http_host;
-}
+        proxy_set_header X-Forwarded-Proto $scheme;
-
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-location @ffsync {
+        proxy_set_header X-Real-IP $remote_addr;
-        uwsgi_pass unix:///run/uwsgi/app/ffsync/socket;
+        proxy_redirect off;
-        include uwsgi_params;
+        proxy_read_timeout 120;
+        proxy_connect_timeout 10;
+        proxy_pass http://127.0.0.1:5000/;
 
-        # Include SSOWAT user panel.
         include conf.d/yunohost_panel.conf.inc;
 }

conf/syncserver.ini

@@ -1,7 +1,9 @@
 [server:main]
-use = egg:Paste#http
+use = egg:gunicorn
-host = 0.0.0.0
+host = 127.0.0.1
 port = 5000
+workers = 2
+timeout = 60
 
 [app:main]
 use = egg:syncserver

scripts/install

@@ -10,8 +10,8 @@ if [[ ! $? -eq 0 ]]; then
     exit 1
 fi
 
-# Generate random password
+# Generate random passworid
-db_pwd=$(dd if=/dev/urandom bs=1 count=200 2> /dev/null | tr -c -d '[A-Za-z0-9]' | sed -n 's/\(.\{24\}\).*/\1/p')
+db_pwd=$(head -c 8 /dev/urandom | sha1sum | cut -d " " -f1)
 
 # Use 'FSyncMS' as database name and user
 db_user=ffsync
@@ -37,6 +37,7 @@ fi
 final_path=/opt/yunohost/ffsync
 sudo mkdir -p $final_path
 sudo cp -a ../sources/* $final_path
+sudo cp ../conf/ffsync /etc/init.d/
 
 # Set permissions to ffsync directory
 sudo useradd ffsync -d $final_path
@@ -59,7 +60,7 @@ sudo sed -i "s/yunopass/$db_pwd/g" $final_path/syncserver.ini
 sudo sed -i "s/yunobase/$db_user/g" $final_path/syncserver.ini
 
 # Init virtualenv
-cd $final_path && sudo make build
+cd $final_path && sudo make build && sudo ./local/bin/easy_install gunicorn
 
 # Disable swapfile
 if [ -f $tmp_swap_file ];
@@ -73,6 +74,11 @@ sudo find $final_path/ -type d -exec chmod 2755 {} \;
 sudo find $final_path/ -type f -exec chmod g+r,o+r {} \;
 sudo usermod -a -G ffsync www-data
 
+#enable services
+sudo chmod +x /etc/init.d/ffsync
+sudo update-rc.d ffsync defaults
+sudo service ffsync start
+
 # Reload Nginx and regenerate SSOwat conf
 sudo service uwsgi restart
 sudo service nginx reload

scripts/upgrade

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+# Retrieve arguments
+domain=$(sudo yunohost app setting searx domain)
+path=$(sudo yunohost app setting searx path)
+
+
+# Check Swap
+tmp_swap_file=/tmp/ffsync_swapfile
+if [ $(sudo swapon -s | wc -l) = 1 ];
+then
+	sudo dd if=/dev/zero of=$tmp_swap_file bs=1M count=256
+	sudo chmod 600 $tmp_swap_file
+	sudo mkswap $tmp_swap_file
+	sudo swapon $tmp_swap_file
+fi
+
+# Copy files to the right place
+final_path=/opt/yunohost/ffsync
+sudo mkdir -p $final_path
+sudo cp -a ../sources/* $final_path
+sudo cp ../conf/ffsync /etc/init.d/
+
+# Set permissions to ffsync directory
+sudo useradd ffsync -d $final_path
+sudo chown ffsync:ffsync -R $final_path
+
+# Modify Nginx configuration file and copy it to Nginx conf directory
+sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf
+sed -i "s@ALIASTOCHANGE@$final_path/@g" ../conf/nginx.conf
+sudo cp ../conf/nginx.conf /etc/nginx/conf.d/$domain.d/ffsync.conf
+
+# Init virtualenv
+cd $final_path && sudo make build && sudo ./local/bin/easy_install gunicorn
+
+# Disable swapfile
+if [ -f $tmp_swap_file ];
+then
+        sudo swapoff $tmp_swap_file
+        sudo rm -f $tmp_swap_file
+fi
+
+# Fix permission
+sudo find $final_path/ -type d -exec chmod 2755 {} \;
+sudo find $final_path/ -type f -exec chmod g+r,o+r {} \;
+sudo usermod -a -G ffsync www-data
+
+#enable services
+sudo chmod +x /etc/init.d/ffsync
+sudo update-rc.d ffsync defaults
+sudo service ffsync start
+
+# Reload Nginx and regenerate SSOwat conf
+sudo service nginx reload
+sudo yunohost app setting ffsync skipped_uris -v "/"
+sudo yunohost app ssowatconf

sources/.gitignore

@@ -1,2 +1,9 @@
-*~
+*.pyc
+*.mako.py
+local
+*.egg-info
 *.swp
+\.coverage
+*~
+nosetests.xml
+syncserver.db

sources/.travis.yml

@@ -0,0 +1,20 @@
+language: python
+
+python:
+  - "2.6"
+  - "2.7"
+
+notifications:
+  email:
+    - rfkelly@mozilla.com
+  irc:
+    channels:
+      - "irc.mozilla.org#services-dev"
+    use_notice: false
+    skip_join: false
+
+install:
+  - make build
+
+script:
+  - make test

sources/requirements.txt

@@ -6,9 +6,7 @@ simplejson==3.4
 SQLAlchemy==0.9.4
 unittest2==0.5.1
 zope.component==4.2.1
+configparser==3.5.0b2
 https://github.com/mozilla-services/mozservices/archive/e00e1b68130423ad98d0f6185655bde650443da8.zip
 https://github.com/mozilla-services/tokenserver/archive/d7e513e8a4f5c588b70d685a8df1d2e508c341c0.zip
 http://github.com/mozilla-services/server-syncstorage/archive/1.5.5.zip
-# Newer releases of configparser have b/w compat bug:
-# https://github.com/mozilla-services/syncserver/issues/39
-configparser==3.3.0r2

sources/syncserver.ini

@@ -27,8 +27,16 @@ public_url = http://localhost:5000/
 # Only request by existing accounts will be honoured.
 # allow_new_users = false
 
+# Set this to "true" to work around a mismatch between public_url and
+# the application URL as seen by python, which can happen in certain reverse-
+# proxy hosting setups.  It will overwrite the WSGI environ dict with the
+# details from public_url.  This could have security implications if e.g.
+# you tell the app that it's on HTTPS but it's really on HTTP, so it should
+# only be used as a last resort and after careful checking of server config.
+force_wsgi_environ = false
+
 # Uncomment and edit the following to use a local BrowserID verifier
-# rather than posing assertions to the mozilla-hosted verifier.
+# rather than posting assertions to the mozilla-hosted verifier.
 # Audiences should be set to your public_url without a trailing slash.
 #[browserid]
 #backend = tokenserver.verifiers.LocalVerifier

sources/syncserver/__init__.py

@@ -117,16 +117,27 @@ def reconcile_wsgi_environ_with_public_url(event):
     # is serving us at some sub-path.
     if not request.script_name:
         request.script_name = p_public_url.path.rstrip("/")
-    # Log a noisy error if the application url is different to what we'd
+    # If the environ does not match public_url, requests are almost certainly
-    # expect based on public_url setting.
+    # going to fail due to auth errors.  We can either bail out early, or we
+    # can forcibly clobber the WSGI environ with the values from public_url.
+    # This is a security risk if you've e.g. mis-configured the server, so
+    # it's not enabled by default.
     application_url = request.application_url
     if public_url != application_url:
-        msg = "The public_url setting does not match the application url.\n"
+        if not request.registry.settings.get("syncserver.force_wsgi_environ"):
-        msg += "This will almost certainly cause authentication failures!\n"
+            msg = "\n".join((
-        msg += "    public_url setting is: %s\n" % (public_url,)
+                "The public_url setting doesn't match the application url.",
-        msg += "    application url is:    %s\n" % (application_url,)
+                "This will almost certainly cause authentication failures!",
-        logger.error(msg)
+                "    public_url setting is: %s" % (public_url,),
-        raise _JSONError([msg], status_code=500)
+                "    application url is:    %s" % (application_url,),
+                "You can disable this check by setting the force_wsgi_environ",
+                "option in your config file, but do so at your own risk.",
+            ))
+            logger.error(msg)
+            raise _JSONError([msg], status_code=500)
+        request.scheme = p_public_url.scheme
+        request.host = p_public_url.netloc
+        request.script_name = p_public_url.path.rstrip("/")
 
 
 def get_configurator(global_config, **settings):

sources/syncserver/staticnode.py

@@ -118,6 +118,9 @@ class StaticNodeAssignment(object):
             if urlparse.urlparse(sqluri).path.lower() in ("/", "/:memory:"):
                 sqlkw["pool_size"] = 1
                 sqlkw["max_overflow"] = 0
+        if "mysql" in self.driver:
+            # Guard against the db closing idle conections.
+            sqlkw["pool_recycle"] = 3600
         self._engine = create_engine(sqluri, **sqlkw)
         users.create(self._engine, checkfirst=True)