From 0d7e8fb9ae24ea35238fee14f9fc7a7d35066e5d Mon Sep 17 00:00:00 2001 From: Adrien Beudin Date: Sat, 15 Nov 2014 16:24:32 +0000 Subject: [PATCH] update ffsync 1.5 --- conf/ffsync.ini | 12 ++ conf/ffsync.sql | 84 ------------ conf/nginx.conf | 26 ++-- conf/settings.php | 26 ---- conf/syncserver.ini | 35 +++++ conf/weave_storage.diff | 39 ------ scripts/install | 60 +++++++-- scripts/remove | 9 +- sources/Dockerfile | 47 +++++++ sources/MANIFEST.in | 3 + sources/Makefile | 50 +++++++ sources/README.rst | 73 +++++++++++ sources/requirements.txt | 13 ++ sources/setup.py | 16 +++ sources/syncserver.ini | 35 +++++ sources/syncserver.wsgi | 40 ++++++ sources/syncserver/__init__.py | 146 +++++++++++++++++++++ sources/syncserver/staticnode.py | 218 +++++++++++++++++++++++++++++++ sources/syncserver/tests.ini | 17 +++ 19 files changed, 771 insertions(+), 178 deletions(-) create mode 100644 conf/ffsync.ini delete mode 100644 conf/ffsync.sql delete mode 100644 conf/settings.php create mode 100644 conf/syncserver.ini delete mode 100644 conf/weave_storage.diff create mode 100644 sources/Dockerfile create mode 100644 sources/MANIFEST.in create mode 100644 sources/Makefile create mode 100644 sources/README.rst create mode 100644 sources/requirements.txt create mode 100644 sources/setup.py create mode 100644 sources/syncserver.ini create mode 100644 sources/syncserver.wsgi create mode 100644 sources/syncserver/__init__.py create mode 100644 sources/syncserver/staticnode.py create mode 100644 sources/syncserver/tests.ini diff --git a/conf/ffsync.ini b/conf/ffsync.ini new file mode 100644 index 0000000..ac209ea --- /dev/null +++ b/conf/ffsync.ini @@ -0,0 +1,12 @@ +[uwsgi] +chmod-socket = 660 +master = true +enable-threads = true +workers = 2 +no-orphans = true +log-date = true +uid = ffsync +gid = ffsync +pythonpath = /opt/yunohost/ffsync/local/lib/python2.7/site-packages +wsgi-file = /opt/yunohost/ffsync/syncserver.wsgi +vacuum = true diff --git a/conf/ffsync.sql b/conf/ffsync.sql deleted file mode 100644 index 3f03fcd..0000000 --- a/conf/ffsync.sql +++ /dev/null @@ -1,84 +0,0 @@ --- MySQL dump 10.13 Distrib 5.5.31, for debian-linux-gnu (x86_64) --- --- Host: localhost Database: ffsync --- ------------------------------------------------------ --- Server version 5.5.31-0+wheezy1 - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; -/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; -/*!40103 SET TIME_ZONE='+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - --- --- Table structure for table `users` --- - -DROP TABLE IF EXISTS `users`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `users` ( - `username` varchar(255) NOT NULL DEFAULT '', - `md5` varchar(124) DEFAULT NULL, - PRIMARY KEY (`username`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `users` --- - -LOCK TABLES `users` WRITE; -/*!40000 ALTER TABLE `users` DISABLE KEYS */; -/*!40000 ALTER TABLE `users` ENABLE KEYS */; -UNLOCK TABLES; - --- --- Table structure for table `wbo` --- - -DROP TABLE IF EXISTS `wbo`; -/*!40101 SET @saved_cs_client = @@character_set_client */; -/*!40101 SET character_set_client = utf8 */; -CREATE TABLE `wbo` ( - `username` varchar(100) NOT NULL DEFAULT '', - `id` varchar(65) NOT NULL DEFAULT '', - `collection` varchar(100) NOT NULL DEFAULT '', - `parentid` varchar(65) DEFAULT NULL, - `predecessorid` int(11) DEFAULT NULL, - `modified` double DEFAULT NULL, - `sortindex` int(11) DEFAULT NULL, - `payload` text, - `payload_size` int(11) DEFAULT NULL, - `ttl` int(11) DEFAULT NULL, - PRIMARY KEY (`username`,`collection`,`id`), - KEY `parentindex` (`username`,`parentid`), - KEY `predecessorindex` (`username`,`predecessorid`), - KEY `modifiedindex` (`username`,`collection`,`modified`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; -/*!40101 SET character_set_client = @saved_cs_client */; - --- --- Dumping data for table `wbo` --- - -LOCK TABLES `wbo` WRITE; -/*!40000 ALTER TABLE `wbo` DISABLE KEYS */; -/*!40000 ALTER TABLE `wbo` ENABLE KEYS */; -UNLOCK TABLES; -/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; - -/*!40101 SET SQL_MODE=@OLD_SQL_MODE */; -/*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; -/*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; -/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; -/*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; -/*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; -/*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; - --- Dump completed on 2013-12-12 11:27:30 diff --git a/conf/nginx.conf b/conf/nginx.conf index 175966b..e8e7538 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,16 +1,14 @@ location PATHTOCHANGE { - alias ALIASTOCHANGE; - if ($scheme = http) { - rewrite ^ https://$server_name$request_uri? permanent; - } - index index.php; - try_files $uri $uri/ /index.php?$args; - location ~ [^/]\.php(/|$) { - fastcgi_split_path_info ^(.+?\.php)(/.*)$; - fastcgi_pass unix:/var/run/php5-fpm.sock; - fastcgi_index index.php; - include fastcgi_params; - fastcgi_param REMOTE_USER $remote_user; - fastcgi_param PATH_INFO $fastcgi_path_info; - } + if ($scheme = http) { + rewrite ^ https://$server_name$request_uri? permanent; + } + try_files $uri @ffsync; +} + +location @ffsync { + uwsgi_pass unix:///run/uwsgi/app/ffsync/socket; + include uwsgi_params; + + # Include SSOWAT user panel. + include conf.d/yunohost_panel.conf.inc; } diff --git a/conf/settings.php b/conf/settings.php deleted file mode 100644 index 17e97b1..0000000 --- a/conf/settings.php +++ /dev/null @@ -1,26 +0,0 @@ - diff --git a/conf/syncserver.ini b/conf/syncserver.ini new file mode 100644 index 0000000..98bb066 --- /dev/null +++ b/conf/syncserver.ini @@ -0,0 +1,35 @@ +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +use = egg:syncserver + +[syncserver] +# This must be edited to point to the public URL of your server, +# i.e. the URL as seen by Firefox. +public_url = https://ynhbaseurl/ + +# This defines the database in which to store all server data. +sqluri = sqlite:////opt/yunohost/ffsync/syncserver.db + +# This is a secret key used for signing authentication tokens. +# It should be long and randomly-generated. +# The following command will give a suitable value on *nix systems: +# +# head -c 20 /dev/urandom | sha1sum +# +# If not specified then the server will generate a temporary one at startup. +#secret = INSERT_SECRET_KEY_HERE + +# Set this to "false" to disable new-user signups on the server. +# Only request by existing accounts will be honoured. +# allow_new_users = false + +# Uncomment and edit the following to use a local BrowserID verifier +# rather than posing assertions to the mozilla-hosted verifier. +# Audiences should be set to your public_url without a trailing slash. +#[browserid] +#backend = tokenserver.verifiers.LocalVerifier +#audiences = https://localhost:5000 diff --git a/conf/weave_storage.diff b/conf/weave_storage.diff deleted file mode 100644 index ac322f7..0000000 --- a/conf/weave_storage.diff +++ /dev/null @@ -1,39 +0,0 @@ ---- weave_storage.php 2013-12-16 13:44:24.252240725 +0000 -+++ weave_storage.php.new 2013-12-16 13:43:15.480260767 +0000 -@@ -720,6 +720,36 @@ - - function create_user($username, $password) - { -+ $mail = $auth_user; -+ $dn = "ou=users,dc=yunohost,dc=org"; -+ $filter = "(&(objectclass=inetOrgPerson)(mail=".$mail."))"; -+ $justthese = array("uid"); -+ // connect to ldap server -+ $ldapconn = ldap_connect("localhost") -+ or die("Could not connect to LDAP server."); -+ -+ ldap_set_option($ldapconn, LDAP_OPT_PROTOCOL_VERSION, 3); -+ ldap_set_option($ldapconn, LDAP_OPT_REFERRALS, 0); -+ -+ if ($ldapconn) { -+ $ldapbind = @ldap_bind($ldapconn); -+ if (! $ldapbind) { -+ log_error("create_user:" . $exception->getMessage()); -+ error_log("create_user:" . $exception->getMessage()); -+ return 0; -+ } -+ else { -+ $sr = ldap_search($ldapconn, $dn, $filter, $justthese); -+ $info = ldap_get_entries($ldapconn, $sr); -+ if ( ! $info["count"]) { -+ log_error("create_user:" . $exception->getMessage()); -+ error_log("create_user:" . $exception->getMessage()); -+ return 0; -+ } -+ } -+ @ldap_unbind($ldapconn); -+ } -+ - log_error("Create User - Username: ".$username."|".$password); - - try diff --git a/scripts/install b/scripts/install index 389a105..9424cf0 100644 --- a/scripts/install +++ b/scripts/install @@ -17,33 +17,65 @@ db_pwd=$(dd if=/dev/urandom bs=1 count=200 2> /dev/null | tr -c -d '[A-Za-z0-9]' db_user=ffsync # Initialize database and store mysql password for upgrade -sudo yunohost app initdb $db_user -p $db_pwd -s $(readlink -e ../conf/ffsync.sql) +sudo yunohost app initdb $db_user -p $db_pwd sudo yunohost app setting ffsync mysqlpwd -v $db_pwd +# Check depends installation +sudo apt-get install python-dev git-core python-virtualenv uwsgi + +# Check Swap +tmp_swap_file=/tmp/ffsync_swapfile +if [ $(sudo swapon -s | wc -l) = 1 ]; +then + sudo dd if=/dev/zero of=$tmp_swap_file bs=1M count=256 + sudo chmod 600 $tmp_swap_file + sudo mkswap $tmp_swap_file + sudo swapon $tmp_swap_file +fi + # Copy files to the right place -final_path=/var/www/ffsync +final_path=/opt/yunohost/ffsync sudo mkdir -p $final_path sudo cp -a ../sources/* $final_path -sudo cp ../conf/settings.php $final_path/ +#sudo cp ../conf/settings.php $final_path/ # Change variables in FSyncMS configuration -sudo sed -i "s/yunouser/$db_user/g" $final_path/settings.php -sudo sed -i "s/yunopass/$db_pwd/g" $final_path/settings.php -sudo sed -i "s/yunobase/$db_user/g" $final_path/settings.php -sudo sed -i "s@URLFFSYNC@$domain$path@g" $final_path/settings.php +#sudo sed -i "s/yunouser/$db_user/g" $final_path/settings.php +#sudo sed -i "s/yunopass/$db_pwd/g" $final_path/settings.php +#sudo sed -i "s/yunobase/$db_user/g" $final_path/settings.php +#sudo sed -i "s@URLFFSYNC@$domain$path@g" $final_path/settings.php + +# Set permissions to ffsync directory +sudo useradd ffsync -d $final_path +sudo chown ffsync:ffsync -R $final_path -# Set permissions to roundcube directory -sudo chown -R www-data: $final_path +# Copy uwsgi config +sudo cp ../conf/ffsync.ini /etc/uwsgi/apps-available/ +sudo ln -s /etc/uwsgi/apps-available/ffsync.ini /etc/uwsgi/apps-enabled/ # Modify Nginx configuration file and copy it to Nginx conf directory -sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf* -sed -i "s@ALIASTOCHANGE@$final_path/@g" ../conf/nginx.conf* +sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf +sed -i "s@ALIASTOCHANGE@$final_path/@g" ../conf/nginx.conf sudo cp ../conf/nginx.conf /etc/nginx/conf.d/$domain.d/ffsync.conf -sudo cp ../conf/weave_storage.diff /tmp/weave_storage.diff -#sudo patch -p0 $final_path/weave_storage.php < /tmp/weave_storage.diff -sudo rm $final_path/setup.php +sudo sed -i -e "s@ynhbaseurl@$domain$path@g" $final_path/syncserver.ini + +# Init virtualenv +cd $final_path && sudo make build + +# Disable swapfile +if [ -f $tmp_swap_file ]; +then + sudo swapoff $tmp_swap_file + sudo rm -f $tmp_swap_file +fi + +# Fix permission +sudo find $final_path/ -type d -exec chmod 2755 {} \; +sudo find $final_path/ -type f -exec chmod g+r,o+r {} \; +sudo usermod -a -G ffsync www-data # Reload Nginx and regenerate SSOwat conf +sudo service uwsgi restart sudo service nginx reload sudo yunohost app setting ffsync skipped_uris -v "/" sudo yunohost app ssowatconf diff --git a/scripts/remove b/scripts/remove index 8dd0d82..6381200 100644 --- a/scripts/remove +++ b/scripts/remove @@ -6,5 +6,12 @@ root_pwd=$(sudo cat /etc/yunohost/mysql) domain=$(sudo yunohost app setting ffsync domain) mysql -u root -p$root_pwd -e "DROP DATABASE $db_name ; DROP USER $db_user@localhost ;" -sudo rm -rf /var/www/ffsync +sudo rm -rf /opt/yunohost/ffsync sudo rm -f /etc/nginx/conf.d/$domain.d/ffsync.conf +sudo rm -f /etc/uwsgi/apps-enabled/searx.ini +sudo rm -f /etc/uwsgi/apps-available/searx.ini + +sudo service uwsgi stop +sudo killall uwsgi +sudo service uwsgi start +sudo userdel ffsync diff --git a/sources/Dockerfile b/sources/Dockerfile new file mode 100644 index 0000000..53208bc --- /dev/null +++ b/sources/Dockerfile @@ -0,0 +1,47 @@ +########################################################## +# /!\ WARNING /!\ # +# This is completely experimental. Use at your own risk. # +# Also, learn you some docker: # +# http://docker.io/gettingstarted # +########################################################## + +FROM debian:7.4 +MAINTAINER Dan Callahan + +# Base system setup + +RUN DEBIAN_FRONTEND=noninteractive apt-get update \ + && apt-get install --no-install-recommends -y \ + vim locales \ + && apt-get clean + +RUN locale-gen C.UTF-8 && LANG=C.UTF-8 /usr/sbin/update-locale + +ENV LANG C.UTF-8 + +RUN useradd --create-home app + +# Build the Sync server + +RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + ca-certificates \ + build-essential \ + libzmq-dev \ + python-dev \ + python-virtualenv \ + && apt-get clean + +USER app + +RUN mkdir -p /home/app/syncserver +ADD ./ /home/app/syncserver +WORKDIR /home/app/syncserver + +RUN make build + +# Run the Sync server + +EXPOSE 5000 + +ENTRYPOINT ["/usr/bin/make"] +CMD ["serve"] diff --git a/sources/MANIFEST.in b/sources/MANIFEST.in new file mode 100644 index 0000000..3ffd5cf --- /dev/null +++ b/sources/MANIFEST.in @@ -0,0 +1,3 @@ +include syncserver.ini +include syncserver.wsgi +include syncserver/tests.ini diff --git a/sources/Makefile b/sources/Makefile new file mode 100644 index 0000000..e9c0d63 --- /dev/null +++ b/sources/Makefile @@ -0,0 +1,50 @@ +SYSTEMPYTHON = `which python2 python | head -n 1` +VIRTUALENV = virtualenv --python=$(SYSTEMPYTHON) +ENV = ./local +TOOLS := $(addprefix $(ENV)/bin/,flake8 nosetests) + +# Hackety-hack around OSX system python bustage. +# The need for this should go away with a future osx/xcode update. +ARCHFLAGS = -Wno-error=unused-command-line-argument-hard-error-in-future + +# Hackety-hack around errors duing compile of ultramemcached. +CFLAGS = -Wno-error + +INSTALL = CFLAGS=$(CFLAGS) ARCHFLAGS=$(ARCHFLAGS) $(ENV)/bin/pip install + + +.PHONY: all +all: build + +.PHONY: build +build: | $(ENV)/COMPLETE +$(ENV)/COMPLETE: requirements.txt + $(VIRTUALENV) --no-site-packages $(ENV) + $(INSTALL) -r requirements.txt + $(ENV)/bin/python ./setup.py develop + touch $(ENV)/COMPLETE + +.PHONY: test +test: | $(TOOLS) + $(ENV)/bin/flake8 ./syncserver + $(ENV)/bin/nosetests -s syncstorage.tests + # Tokenserver tests currently broken due to incorrect file paths + # $(ENV)/bin/nosetests -s tokenserver.tests + + # Test against a running server + $(ENV)/bin/pserve syncserver/tests.ini 2> /dev/null & SERVER_PID=$$!; \ + sleep 2; \ + $(ENV)/bin/python -m syncstorage.tests.functional.test_storage \ + --use-token-server http://localhost:5000/token/1.0/sync/1.5; \ + kill $$SERVER_PID + +$(TOOLS): | $(ENV)/COMPLETE + $(INSTALL) nose flake8 + +.PHONY: serve +serve: | $(ENV)/COMPLETE + $(ENV)/bin/pserve ./syncserver.ini + +.PHONY: clean +clean: + rm -rf $(ENV) diff --git a/sources/README.rst b/sources/README.rst new file mode 100644 index 0000000..3872786 --- /dev/null +++ b/sources/README.rst @@ -0,0 +1,73 @@ +Run-Your-Own Firefox Sync Server +================================ + +This is an all-in-one package for running a self-hosted Firefox Sync server. +If bundles the "tokenserver" project for authentication and the "syncstorage" +project for storage, produce a single stand-alone webapp. + +Complete installation instructions are available at: + + https://docs.services.mozilla.com/howtos/run-sync-1.5.html + + +Quickstart +---------- + +The Sync Server software runs using **python 2.6** or later, and the build +process requires **make** and **virtualenv**. You will need to have the +following packages (or similar, depending on your operating system) installed: + +- python2.7 +- python2.7-dev +- python-virtualenv +- make + +Take a checkout of this repository, then run "make build" to pull in the +necessary python package dependencies:: + + $ git clone https://github.com/mozilla-services/syncserver + $ cd syncserver + $ make build + +To sanity-check that things got installed correctly, do the following:: + + $ make test + +Now you can run the server:: + + $ make serve + +This should start a server on http://localhost:5000/. + +Now go into Firefox's `about:config` page, search for a setting named +"tokenServerURI", and change it to point to your server:: + + services.sync.tokenServerURI: http://localhost:5000/token/1.0/sync/1.5 + +Firefox should now sync against your local server rather than the default +Mozilla-hosted servers. + +For more details on setting up a stable deployment, see: + + https://docs.services.mozilla.com/howtos/run-sync-1.5.html + + +Customization +------------- + +All customization of the server can be done by editing the file +"syncserver.ini", which contains lots of comments to help you on +your way. Things you might like to change include: + + * The client-visible hostname for your server. Edit the "public_url" + key under the [syncstorage] section. + + * The database in which to store sync data. Edit the "sqluri" setting + under the [syncstorage] section. + + +Questions, Feedback +------------------- + +- IRC channel: #sync. See http://irc.mozilla.org/ +- Mailing list: https://mail.mozilla.org/listinfo/services-dev diff --git a/sources/requirements.txt b/sources/requirements.txt new file mode 100644 index 0000000..39b7025 --- /dev/null +++ b/sources/requirements.txt @@ -0,0 +1,13 @@ +cornice==0.16.2 +gunicorn==19.1.1 +pyramid==1.5 +requests==2.2.1 +simplejson==3.4 +SQLAlchemy==0.9.4 +unittest2==0.5.1 +https://github.com/mozilla-services/mozservices/archive/e00e1b68130423ad98d0f6185655bde650443da8.zip +https://github.com/mozilla-services/tokenserver/archive/d7e513e8a4f5c588b70d685a8df1d2e508c341c0.zip +http://github.com/mozilla-services/server-syncstorage/archive/1.5.5.zip +# Newer releases of configparser have b/w compat bug: +# https://github.com/mozilla-services/syncserver/issues/39 +configparser==3.3.0r2 diff --git a/sources/setup.py b/sources/setup.py new file mode 100644 index 0000000..76d6405 --- /dev/null +++ b/sources/setup.py @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +from setuptools import setup + +entry_points = """ +[paste.app_factory] +main = syncserver:main +""" + +setup( + name='syncserver', + version="1.5.1", + packages=['syncserver'], + entry_points=entry_points +) diff --git a/sources/syncserver.ini b/sources/syncserver.ini new file mode 100644 index 0000000..63cb2d5 --- /dev/null +++ b/sources/syncserver.ini @@ -0,0 +1,35 @@ +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +use = egg:syncserver + +[syncserver] +# This must be edited to point to the public URL of your server, +# i.e. the URL as seen by Firefox. +public_url = http://localhost:5000/ + +# This defines the database in which to store all server data. +#sqluri = sqlite:////tmp/syncserver.db + +# This is a secret key used for signing authentication tokens. +# It should be long and randomly-generated. +# The following command will give a suitable value on *nix systems: +# +# head -c 20 /dev/urandom | sha1sum +# +# If not specified then the server will generate a temporary one at startup. +#secret = INSERT_SECRET_KEY_HERE + +# Set this to "false" to disable new-user signups on the server. +# Only request by existing accounts will be honoured. +# allow_new_users = false + +# Uncomment and edit the following to use a local BrowserID verifier +# rather than posing assertions to the mozilla-hosted verifier. +# Audiences should be set to your public_url without a trailing slash. +#[browserid] +#backend = tokenserver.verifiers.LocalVerifier +#audiences = https://localhost:5000 diff --git a/sources/syncserver.wsgi b/sources/syncserver.wsgi new file mode 100644 index 0000000..4b0503c --- /dev/null +++ b/sources/syncserver.wsgi @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import sys +import site +from logging.config import fileConfig +from ConfigParser import NoSectionError + +# detecting if virtualenv was used in this dir +_CURDIR = os.path.dirname(os.path.abspath(__file__)) +_PY_VER = sys.version.split()[0][:3] +_SITE_PKG = os.path.join(_CURDIR, 'local', 'lib', 'python' + _PY_VER, 'site-packages') + +# adding virtualenv's site-package and ordering paths +saved = sys.path[:] + +if os.path.exists(_SITE_PKG): + site.addsitedir(_SITE_PKG) + +for path in sys.path: + if path not in saved: + saved.insert(0, path) + +sys.path[:] = saved + +# setting up the egg cache to a place where apache can write +os.environ['PYTHON_EGG_CACHE'] = '/tmp/python-eggs' + +# setting up logging +ini_file = os.path.join(_CURDIR, 'syncserver.ini') +try: + fileConfig(ini_file) +except NoSectionError: + pass + +# running the app using Paste +from paste.deploy import loadapp +application = loadapp('config:%s'% ini_file) diff --git a/sources/syncserver/__init__.py b/sources/syncserver/__init__.py new file mode 100644 index 0000000..eb90f6a --- /dev/null +++ b/sources/syncserver/__init__.py @@ -0,0 +1,146 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. + +import os +import logging +from urlparse import urlparse, urlunparse + +from pyramid.response import Response +from pyramid.events import NewRequest, subscriber + +import mozsvc.config + +from tokenserver.util import _JSONError + +logger = logging.getLogger("syncserver") + + +def includeme(config): + """Install SyncServer application into the given Pyramid configurator.""" + # Set the umask so that files are created with secure permissions. + # Necessary for e.g. created-on-demand sqlite database files. + os.umask(0077) + + # Sanity-check the deployment settings and provide sensible defaults. + settings = config.registry.settings + public_url = settings.get("syncserver.public_url") + if public_url is None: + raise RuntimeError("you much configure syncserver.public_url") + public_url = public_url.rstrip("/") + settings["syncserver.public_url"] = public_url + + secret = settings.get("syncserver.secret") + if secret is None: + secret = os.urandom(32).encode("hex") + sqluri = settings.get("syncserver.sqluri") + if sqluri is None: + rootdir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + sqluri = "sqlite:///" + os.path.join(rootdir, "syncserver.db") + + # Configure app-specific defaults based on top-level configuration. + settings.pop("config", None) + if "tokenserver.backend" not in settings: + # Default to our simple static node-assignment backend + settings["tokenserver.backend"] =\ + "syncserver.staticnode.StaticNodeAssignment" + settings["tokenserver.sqluri"] = sqluri + settings["tokenserver.node_url"] = public_url + settings["endpoints.sync-1.5"] = "{node}/storage/1.5/{uid}" + if "tokenserver.monkey_patch_gevent" not in settings: + # Default to no gevent monkey-patching + settings["tokenserver.monkey_patch_gevent"] = False + if "tokenserver.applications" not in settings: + # Default to just the sync-1.5 application + settings["tokenserver.applications"] = "sync-1.5" + if "tokenserver.secrets.backend" not in settings: + # Default to a single fixed signing secret + settings["tokenserver.secrets.backend"] = "mozsvc.secrets.FixedSecrets" + settings["tokenserver.secrets.secrets"] = [secret] + if "tokenserver.allow_new_users" not in settings: + allow_new_users = settings.get("syncserver.allow_new_users") + if allow_new_users is not None: + settings["tokenserver.allow_new_users"] = allow_new_users + if "hawkauth.secrets.backend" not in settings: + # Default to the same secrets backend as the tokenserver + for key in settings.keys(): + if key.startswith("tokenserver.secrets."): + newkey = "hawkauth" + key[len("tokenserver"):] + settings[newkey] = settings[key] + if "storage.backend" not in settings: + # Default to sql syncstorage backend + settings["storage.backend"] = "syncstorage.storage.sql.SQLStorage" + settings["storage.sqluri"] = sqluri + settings["storage.create_tables"] = True + if "browserid.backend" not in settings: + # Default to remote verifier, with base of public_url as only audience + audience = urlunparse(urlparse(public_url)._replace(path="")) + settings["browserid.backend"] = "tokenserver.verifiers.RemoteVerifier" + settings["browserid.audiences"] = audience + if "loggers" not in settings: + # Default to basic logging config. + root_logger = logging.getLogger("") + if not root_logger.handlers: + logging.basicConfig(level=logging.INFO) + + # Include the relevant sub-packages. + config.scan("syncserver") + config.include("syncstorage", route_prefix="/storage") + config.include("tokenserver", route_prefix="/token") + + # Add a top-level "it works!" view. + def itworks(request): + return Response("it works!") + + config.add_route('itworks', '/') + config.add_view(itworks, route_name='itworks') + + +@subscriber(NewRequest) +def reconcile_wsgi_environ_with_public_url(event): + """Event-listener that checks and tweaks WSGI environ based on public_url. + + This is a simple trick to help ensure that the configured public_url + matches the actual deployed address. It fixes fixes parts of the WSGI + environ where it makes sense (e.g. SCRIPT_NAME) and warns about any parts + that seem obviously mis-configured (e.g. http:// versus https://). + + It's very important to get public_url and WSGI environ matching exactly, + since they're used for browserid audience checking and HAWK signature + validation, so mismatches can easily cause strange and cryptic errors. + """ + request = event.request + public_url = request.registry.settings["syncserver.public_url"] + p_public_url = urlparse(public_url) + # If we don't have a SCRIPT_NAME, take it from the public_url. + # This is often the case if we're behind e.g. an nginx proxy that + # is serving us at some sub-path. + if not request.script_name: + request.script_name = p_public_url.path.rstrip("/") + # Log a noisy error if the application url is different to what we'd + # expect based on public_url setting. + application_url = request.application_url + if public_url != application_url: + msg = "The public_url setting does not match the application url.\n" + msg += "This will almost certainly cause authentication failures!\n" + msg += " public_url setting is: %s\n" % (public_url,) + msg += " application url is: %s\n" % (application_url,) + logger.error(msg) + raise _JSONError([msg], status_code=500) + + +def get_configurator(global_config, **settings): + """Load a SyncStorge configurator object from deployment settings.""" + config = mozsvc.config.get_configurator(global_config, **settings) + config.begin() + try: + config.include(includeme) + finally: + config.end() + return config + + +def main(global_config, **settings): + """Load a SyncStorage WSGI app from deployment settings.""" + config = get_configurator(global_config, **settings) + return config.make_wsgi_app() diff --git a/sources/syncserver/staticnode.py b/sources/syncserver/staticnode.py new file mode 100644 index 0000000..d758aad --- /dev/null +++ b/sources/syncserver/staticnode.py @@ -0,0 +1,218 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Simple node-assignment backend using a single, static node. + +This is a greatly-simplified node-assignment backend. It keeps user records +in an SQL database, but does not attempt to do any node management. All users +are implicitly assigned to a single, static node. + +XXX TODO: move this into the tokenserver repo. + +""" +import time +import urlparse +from mozsvc.exceptions import BackendError + +from sqlalchemy import Column, Integer, String, BigInteger, Index +from sqlalchemy import create_engine, Table, MetaData +from sqlalchemy.pool import QueuePool +from sqlalchemy.sql import text as sqltext +from sqlalchemy.exc import IntegrityError + +from tokenserver.assignment import INodeAssignment +from zope.interface import implements + + +metadata = MetaData() + + +users = Table( + "users", + metadata, + Column("uid", Integer(), primary_key=True, autoincrement=True, + nullable=False), + Column("service", String(32), nullable=False), + Column("email", String(255), nullable=False), + Column("generation", BigInteger(), nullable=False), + Column("client_state", String(32), nullable=False), + Column("created_at", BigInteger(), nullable=False), + Column("replaced_at", BigInteger(), nullable=True), + Index('lookup_idx', 'email', 'service', 'created_at'), + Index('clientstate_idx', 'email', 'service', 'client_state', unique=True), +) + + +_GET_USER_RECORDS = sqltext("""\ +select + uid, generation, client_state +from + users +where + email = :email +and + service = :service +order by + created_at desc, uid desc +limit + 20 +""") + + +_CREATE_USER_RECORD = sqltext("""\ +insert into + users + (service, email, generation, client_state, created_at, replaced_at) +values + (:service, :email, :generation, :client_state, :timestamp, NULL) +""") + + +_UPDATE_GENERATION_NUMBER = sqltext("""\ +update + users +set + generation = :generation +where + service = :service and email = :email and + generation < :generation and replaced_at is null +""") + + +_REPLACE_USER_RECORDS = sqltext("""\ +update + users +set + replaced_at = :timestamp +where + service = :service and email = :email + and replaced_at is null and created_at < :timestamp +""") + + +def get_timestamp(): + """Get current timestamp in milliseconds.""" + return int(time.time() * 1000) + + +class StaticNodeAssignment(object): + implements(INodeAssignment) + + def __init__(self, sqluri, node_url, **kw): + self.sqluri = sqluri + self.node_url = node_url + self.driver = urlparse.urlparse(sqluri).scheme.lower() + sqlkw = { + "logging_name": "syncserver", + "connect_args": {}, + "poolclass": QueuePool, + "pool_reset_on_return": True, + } + if self.driver == "sqlite": + # We must mark it as safe to share sqlite connections between + # threads. The pool will ensure there's on race conditions. + sqlkw["connect_args"]["check_same_thread"] = False + # If using a :memory: database, we must use a QueuePool of size + # 1 so that a single connection is shared by all threads. + if urlparse.urlparse(sqluri).path.lower() in ("/", "/:memory:"): + sqlkw["pool_size"] = 1 + sqlkw["max_overflow"] = 0 + self._engine = create_engine(sqluri, **sqlkw) + users.create(self._engine, checkfirst=True) + + def get_user(self, service, email): + params = {'service': service, 'email': email} + res = self._engine.execute(_GET_USER_RECORDS, **params) + try: + row = res.fetchone() + if row is None: + return None + # The first row is the most up-to-date user record. + user = { + 'email': email, + 'uid': row.uid, + 'node': self.node_url, + 'generation': row.generation, + 'client_state': row.client_state, + 'old_client_states': {} + } + # Any subsequent rows are due to old client-state values. + row = res.fetchone() + while row is not None: + user['old_client_states'][row.client_state] = True + row = res.fetchone() + return user + finally: + res.close() + + def allocate_user(self, service, email, generation=0, client_state=''): + params = { + 'service': service, 'email': email, 'generation': generation, + 'client_state': client_state, 'timestamp': get_timestamp() + } + try: + res = self._engine.execute(_CREATE_USER_RECORD, **params) + except IntegrityError: + raise + return self.get_user(service, email) + else: + res.close() + return { + 'email': email, + 'uid': res.lastrowid, + 'node': self.node_url, + 'generation': generation, + 'client_state': client_state, + 'old_client_states': {} + } + + def update_user(self, service, user, generation=None, client_state=None): + if client_state is None: + # uid can stay the same, just update the generation number. + if generation is not None: + params = { + 'service': service, + 'email': user['email'], + 'generation': generation, + } + res = self._engine.execute(_UPDATE_GENERATION_NUMBER, **params) + res.close() + user['generation'] = max(generation, user['generation']) + else: + # reject previously-seen client-state strings. + if client_state == user['client_state']: + raise BackendError('previously seen client-state string') + if client_state in user['old_client_states']: + raise BackendError('previously seen client-state string') + # need to create a new record for new client_state. + if generation is not None: + generation = max(user['generation'], generation) + else: + generation = user['generation'] + now = get_timestamp() + params = { + 'service': service, 'email': user['email'], + 'generation': generation, 'client_state': client_state, + 'timestamp': now, + } + try: + res = self._engine.execute(_CREATE_USER_RECORD, **params) + except IntegrityError: + user.update(self.get_user(service, user['email'])) + else: + self.get_user(service, user['email']) + user['uid'] = res.lastrowid + user['generation'] = generation + user['old_client_states'][user['client_state']] = True + user['client_state'] = client_state + res.close() + # mark old records as having been replaced. + # if we crash here, they are unmarked and we may fail to + # garbage collect them for a while, but the active state + # will be undamaged. + params = { + 'service': service, 'email': user['email'], 'timestamp': now + } + res = self._engine.execute(_REPLACE_USER_RECORDS, **params) + res.close() diff --git a/sources/syncserver/tests.ini b/sources/syncserver/tests.ini new file mode 100644 index 0000000..50b7776 --- /dev/null +++ b/sources/syncserver/tests.ini @@ -0,0 +1,17 @@ +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5000 + +[app:main] +use = egg:SyncServer + +[syncserver] +# This must be edited to point to the public URL of your server. +public_url = http://localhost:5000/ + +# This defines the database in which to store all server data. +#sqluri = sqlite:////tmp/syncserver.db + +# This is a secret key used for signing authentication tokens. +#secret = INSERT_SECRET_KEY_HERE