From c53e73afd69ca48621c351f430b153c949a479e5 Mon Sep 17 00:00:00 2001 From: Adrien Beudin Date: Fri, 13 Feb 2015 18:10:02 +0100 Subject: [PATCH] firefox sync with gunicorn --- conf/ffsync | 64 ++++++++++++++++++++++++++++++++ conf/nginx.conf | 15 ++++---- conf/syncserver.ini | 6 ++- scripts/install | 12 ++++-- scripts/upgrade | 56 ++++++++++++++++++++++++++++ sources/.gitignore | 9 ++++- sources/.travis.yml | 20 ++++++++++ sources/requirements.txt | 4 +- sources/syncserver.ini | 10 ++++- sources/syncserver/__init__.py | 27 ++++++++++---- sources/syncserver/staticnode.py | 3 ++ 11 files changed, 201 insertions(+), 25 deletions(-) create mode 100755 conf/ffsync create mode 100644 scripts/upgrade create mode 100644 sources/.travis.yml diff --git a/conf/ffsync b/conf/ffsync new file mode 100755 index 0000000..8131031 --- /dev/null +++ b/conf/ffsync @@ -0,0 +1,64 @@ +#!/bin/bash -x +# /etc/init.d/sync +# version 0.1 2013-03-12 (YYYY-MM-DD) + +### BEGIN INIT INFO +# Provides: sync +# Required-Start: $local_fs $remote_fs +# Required-Stop: $local_fs $remote_fs +# Should-Start: $network +# Should-Stop: $network +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Mozilla Sync server +# Description: Starts the mozilla sync server +### END INIT INFO + +# Source function library. +#. /etc/rc.d/init.d/functions + +prog=sync +SYNC_USER=ffsync +SYNC_HOME=/opt/yunohost/ffsync +CPU_COUNT=2 +pidfile=/tmp/sync.pid +lockfile=/var/run/mozilla/sync.lock +conffile=${SYNC_HOME}/syncserver.ini +GUNICORN=${SYNC_HOME}/local/bin/gunicorn_paster +GUNICORN_ARGS="--access-logfile /var/log/ffsync.log --daemon -p $pidfile" +#INVOCATION="$GUNICORN $GUNICORN_ARGS $conffile" + +start () { + echo -n "Starting $prog" + start-stop-daemon --start -c ${SYNC_USER} --exec $GUNICORN -- $GUNICORN_ARGS $conffile + RETVAL=$? + echo + [ $RETVAL = 0 ] && touch ${lockfile} + return $RETVAL +} + +stop() { + echo "Stopping $prog" + start-stop-daemon --stop --quiet --oknodo --pidfile ${pidfile} + #log_end_msg $? + rm -f ${pidfile} +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart) + stop + start + ;; + *) + echo $"Usage: $prog {start|stop|restart|help}" + RETVAL=2 +esac + +exit $RETVAL + diff --git a/conf/nginx.conf b/conf/nginx.conf index e8e7538..ab6e584 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -2,13 +2,14 @@ location PATHTOCHANGE { if ($scheme = http) { rewrite ^ https://$server_name$request_uri? permanent; } - try_files $uri @ffsync; -} - -location @ffsync { - uwsgi_pass unix:///run/uwsgi/app/ffsync/socket; - include uwsgi_params; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; + proxy_redirect off; + proxy_read_timeout 120; + proxy_connect_timeout 10; + proxy_pass http://127.0.0.1:5000/; - # Include SSOWAT user panel. include conf.d/yunohost_panel.conf.inc; } diff --git a/conf/syncserver.ini b/conf/syncserver.ini index 9cdf9f1..95ec708 100644 --- a/conf/syncserver.ini +++ b/conf/syncserver.ini @@ -1,7 +1,9 @@ [server:main] -use = egg:Paste#http -host = 0.0.0.0 +use = egg:gunicorn +host = 127.0.0.1 port = 5000 +workers = 2 +timeout = 60 [app:main] use = egg:syncserver diff --git a/scripts/install b/scripts/install index c26a979..047c998 100644 --- a/scripts/install +++ b/scripts/install @@ -10,8 +10,8 @@ if [[ ! $? -eq 0 ]]; then exit 1 fi -# Generate random password -db_pwd=$(dd if=/dev/urandom bs=1 count=200 2> /dev/null | tr -c -d '[A-Za-z0-9]' | sed -n 's/\(.\{24\}\).*/\1/p') +# Generate random passworid +db_pwd=$(head -c 8 /dev/urandom | sha1sum | cut -d " " -f1) # Use 'FSyncMS' as database name and user db_user=ffsync @@ -37,6 +37,7 @@ fi final_path=/opt/yunohost/ffsync sudo mkdir -p $final_path sudo cp -a ../sources/* $final_path +sudo cp ../conf/ffsync /etc/init.d/ # Set permissions to ffsync directory sudo useradd ffsync -d $final_path @@ -59,7 +60,7 @@ sudo sed -i "s/yunopass/$db_pwd/g" $final_path/syncserver.ini sudo sed -i "s/yunobase/$db_user/g" $final_path/syncserver.ini # Init virtualenv -cd $final_path && sudo make build +cd $final_path && sudo make build && sudo ./local/bin/easy_install gunicorn # Disable swapfile if [ -f $tmp_swap_file ]; @@ -73,6 +74,11 @@ sudo find $final_path/ -type d -exec chmod 2755 {} \; sudo find $final_path/ -type f -exec chmod g+r,o+r {} \; sudo usermod -a -G ffsync www-data +#enable services +sudo chmod +x /etc/init.d/ffsync +sudo update-rc.d ffsync defaults +sudo service ffsync start + # Reload Nginx and regenerate SSOwat conf sudo service uwsgi restart sudo service nginx reload diff --git a/scripts/upgrade b/scripts/upgrade new file mode 100644 index 0000000..d37e350 --- /dev/null +++ b/scripts/upgrade @@ -0,0 +1,56 @@ +#!/bin/bash + +# Retrieve arguments +domain=$(sudo yunohost app setting searx domain) +path=$(sudo yunohost app setting searx path) + + +# Check Swap +tmp_swap_file=/tmp/ffsync_swapfile +if [ $(sudo swapon -s | wc -l) = 1 ]; +then + sudo dd if=/dev/zero of=$tmp_swap_file bs=1M count=256 + sudo chmod 600 $tmp_swap_file + sudo mkswap $tmp_swap_file + sudo swapon $tmp_swap_file +fi + +# Copy files to the right place +final_path=/opt/yunohost/ffsync +sudo mkdir -p $final_path +sudo cp -a ../sources/* $final_path +sudo cp ../conf/ffsync /etc/init.d/ + +# Set permissions to ffsync directory +sudo useradd ffsync -d $final_path +sudo chown ffsync:ffsync -R $final_path + +# Modify Nginx configuration file and copy it to Nginx conf directory +sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf +sed -i "s@ALIASTOCHANGE@$final_path/@g" ../conf/nginx.conf +sudo cp ../conf/nginx.conf /etc/nginx/conf.d/$domain.d/ffsync.conf + +# Init virtualenv +cd $final_path && sudo make build && sudo ./local/bin/easy_install gunicorn + +# Disable swapfile +if [ -f $tmp_swap_file ]; +then + sudo swapoff $tmp_swap_file + sudo rm -f $tmp_swap_file +fi + +# Fix permission +sudo find $final_path/ -type d -exec chmod 2755 {} \; +sudo find $final_path/ -type f -exec chmod g+r,o+r {} \; +sudo usermod -a -G ffsync www-data + +#enable services +sudo chmod +x /etc/init.d/ffsync +sudo update-rc.d ffsync defaults +sudo service ffsync start + +# Reload Nginx and regenerate SSOwat conf +sudo service nginx reload +sudo yunohost app setting ffsync skipped_uris -v "/" +sudo yunohost app ssowatconf diff --git a/sources/.gitignore b/sources/.gitignore index b72f9be..45b872e 100644 --- a/sources/.gitignore +++ b/sources/.gitignore @@ -1,2 +1,9 @@ -*~ +*.pyc +*.mako.py +local +*.egg-info *.swp +\.coverage +*~ +nosetests.xml +syncserver.db diff --git a/sources/.travis.yml b/sources/.travis.yml new file mode 100644 index 0000000..2e45ae4 --- /dev/null +++ b/sources/.travis.yml @@ -0,0 +1,20 @@ +language: python + +python: + - "2.6" + - "2.7" + +notifications: + email: + - rfkelly@mozilla.com + irc: + channels: + - "irc.mozilla.org#services-dev" + use_notice: false + skip_join: false + +install: + - make build + +script: + - make test diff --git a/sources/requirements.txt b/sources/requirements.txt index fffde2a..c1a19f9 100644 --- a/sources/requirements.txt +++ b/sources/requirements.txt @@ -6,9 +6,7 @@ simplejson==3.4 SQLAlchemy==0.9.4 unittest2==0.5.1 zope.component==4.2.1 +configparser==3.5.0b2 https://github.com/mozilla-services/mozservices/archive/e00e1b68130423ad98d0f6185655bde650443da8.zip https://github.com/mozilla-services/tokenserver/archive/d7e513e8a4f5c588b70d685a8df1d2e508c341c0.zip http://github.com/mozilla-services/server-syncstorage/archive/1.5.5.zip -# Newer releases of configparser have b/w compat bug: -# https://github.com/mozilla-services/syncserver/issues/39 -configparser==3.3.0r2 diff --git a/sources/syncserver.ini b/sources/syncserver.ini index 63cb2d5..aa7105f 100644 --- a/sources/syncserver.ini +++ b/sources/syncserver.ini @@ -27,8 +27,16 @@ public_url = http://localhost:5000/ # Only request by existing accounts will be honoured. # allow_new_users = false +# Set this to "true" to work around a mismatch between public_url and +# the application URL as seen by python, which can happen in certain reverse- +# proxy hosting setups. It will overwrite the WSGI environ dict with the +# details from public_url. This could have security implications if e.g. +# you tell the app that it's on HTTPS but it's really on HTTP, so it should +# only be used as a last resort and after careful checking of server config. +force_wsgi_environ = false + # Uncomment and edit the following to use a local BrowserID verifier -# rather than posing assertions to the mozilla-hosted verifier. +# rather than posting assertions to the mozilla-hosted verifier. # Audiences should be set to your public_url without a trailing slash. #[browserid] #backend = tokenserver.verifiers.LocalVerifier diff --git a/sources/syncserver/__init__.py b/sources/syncserver/__init__.py index eb90f6a..696a430 100644 --- a/sources/syncserver/__init__.py +++ b/sources/syncserver/__init__.py @@ -117,16 +117,27 @@ def reconcile_wsgi_environ_with_public_url(event): # is serving us at some sub-path. if not request.script_name: request.script_name = p_public_url.path.rstrip("/") - # Log a noisy error if the application url is different to what we'd - # expect based on public_url setting. + # If the environ does not match public_url, requests are almost certainly + # going to fail due to auth errors. We can either bail out early, or we + # can forcibly clobber the WSGI environ with the values from public_url. + # This is a security risk if you've e.g. mis-configured the server, so + # it's not enabled by default. application_url = request.application_url if public_url != application_url: - msg = "The public_url setting does not match the application url.\n" - msg += "This will almost certainly cause authentication failures!\n" - msg += " public_url setting is: %s\n" % (public_url,) - msg += " application url is: %s\n" % (application_url,) - logger.error(msg) - raise _JSONError([msg], status_code=500) + if not request.registry.settings.get("syncserver.force_wsgi_environ"): + msg = "\n".join(( + "The public_url setting doesn't match the application url.", + "This will almost certainly cause authentication failures!", + " public_url setting is: %s" % (public_url,), + " application url is: %s" % (application_url,), + "You can disable this check by setting the force_wsgi_environ", + "option in your config file, but do so at your own risk.", + )) + logger.error(msg) + raise _JSONError([msg], status_code=500) + request.scheme = p_public_url.scheme + request.host = p_public_url.netloc + request.script_name = p_public_url.path.rstrip("/") def get_configurator(global_config, **settings): diff --git a/sources/syncserver/staticnode.py b/sources/syncserver/staticnode.py index d758aad..560d3ac 100644 --- a/sources/syncserver/staticnode.py +++ b/sources/syncserver/staticnode.py @@ -118,6 +118,9 @@ class StaticNodeAssignment(object): if urlparse.urlparse(sqluri).path.lower() in ("/", "/:memory:"): sqlkw["pool_size"] = 1 sqlkw["max_overflow"] = 0 + if "mysql" in self.driver: + # Guard against the db closing idle conections. + sqlkw["pool_recycle"] = 3600 self._engine = create_engine(sqluri, **sqlkw) users.create(self._engine, checkfirst=True)