bwt/ffsync_page
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

update ffsync 1.5

Browse Source
This commit is contained in:
Adrien Beudin
2014-11-15 16:24:32 +00:00
parent 363f126eeb
commit 0d7e8fb9ae
19 changed files with 771 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
sources/Dockerfile Normal file
View File

@@ -0,0 +1,47 @@
##########################################################
# /!\ WARNING /!\ #
# This is completely experimental. Use at your own risk. #
# Also, learn you some docker: #
# http://docker.io/gettingstarted #
##########################################################
FROM debian:7.4
MAINTAINER Dan Callahan <dan.callahan@gmail.com>
# Base system setup
RUN DEBIAN_FRONTEND=noninteractive apt-get update \
&& apt-get install --no-install-recommends -y \
vim locales \
&& apt-get clean
RUN locale-gen C.UTF-8 && LANG=C.UTF-8 /usr/sbin/update-locale
ENV LANG C.UTF-8
RUN useradd --create-home app
# Build the Sync server
RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
ca-certificates \
build-essential \
libzmq-dev \
python-dev \
python-virtualenv \
&& apt-get clean
USER app
RUN mkdir -p /home/app/syncserver
ADD ./ /home/app/syncserver
WORKDIR /home/app/syncserver
RUN make build
# Run the Sync server
EXPOSE 5000
ENTRYPOINT ["/usr/bin/make"]
CMD ["serve"]

3
sources/MANIFEST.in Normal file
View File

@@ -0,0 +1,3 @@
include syncserver.ini
include syncserver.wsgi
include syncserver/tests.ini

50
sources/Makefile Normal file
View File

@@ -0,0 +1,50 @@
SYSTEMPYTHON = `which python2 python | head -n 1`
VIRTUALENV = virtualenv --python=$(SYSTEMPYTHON)
ENV = ./local
TOOLS := $(addprefix $(ENV)/bin/,flake8 nosetests)
# Hackety-hack around OSX system python bustage.
# The need for this should go away with a future osx/xcode update.
ARCHFLAGS = -Wno-error=unused-command-line-argument-hard-error-in-future
# Hackety-hack around errors duing compile of ultramemcached.
CFLAGS = -Wno-error
INSTALL = CFLAGS=$(CFLAGS) ARCHFLAGS=$(ARCHFLAGS) $(ENV)/bin/pip install
.PHONY: all
all: build
.PHONY: build
build: | $(ENV)/COMPLETE
$(ENV)/COMPLETE: requirements.txt
$(VIRTUALENV) --no-site-packages $(ENV)
$(INSTALL) -r requirements.txt
$(ENV)/bin/python ./setup.py develop
touch $(ENV)/COMPLETE
.PHONY: test
test: | $(TOOLS)
$(ENV)/bin/flake8 ./syncserver
$(ENV)/bin/nosetests -s syncstorage.tests
# Tokenserver tests currently broken due to incorrect file paths
# $(ENV)/bin/nosetests -s tokenserver.tests
# Test against a running server
$(ENV)/bin/pserve syncserver/tests.ini 2> /dev/null & SERVER_PID=$$!; \
sleep 2; \
$(ENV)/bin/python -m syncstorage.tests.functional.test_storage \
--use-token-server http://localhost:5000/token/1.0/sync/1.5; \
kill $$SERVER_PID
$(TOOLS): | $(ENV)/COMPLETE
$(INSTALL) nose flake8
.PHONY: serve
serve: | $(ENV)/COMPLETE
$(ENV)/bin/pserve ./syncserver.ini
.PHONY: clean
clean:
rm -rf $(ENV)

73
sources/README.rst Normal file
View File

@@ -0,0 +1,73 @@
Run-Your-Own Firefox Sync Server
================================
This is an all-in-one package for running a self-hosted Firefox Sync server.
If bundles the "tokenserver" project for authentication and the "syncstorage"
project for storage, produce a single stand-alone webapp.
Complete installation instructions are available at:
https://docs.services.mozilla.com/howtos/run-sync-1.5.html
Quickstart
----------
The Sync Server software runs using **python 2.6** or later, and the build
process requires **make** and **virtualenv**. You will need to have the
following packages (or similar, depending on your operating system) installed:
- python2.7
- python2.7-dev
- python-virtualenv
- make
Take a checkout of this repository, then run "make build" to pull in the
necessary python package dependencies::
$ git clone https://github.com/mozilla-services/syncserver
$ cd syncserver
$ make build
To sanity-check that things got installed correctly, do the following::
$ make test
Now you can run the server::
$ make serve
This should start a server on http://localhost:5000/.
Now go into Firefox's `about:config` page, search for a setting named
"tokenServerURI", and change it to point to your server::
services.sync.tokenServerURI: http://localhost:5000/token/1.0/sync/1.5
Firefox should now sync against your local server rather than the default
Mozilla-hosted servers.
For more details on setting up a stable deployment, see:
https://docs.services.mozilla.com/howtos/run-sync-1.5.html
Customization
-------------
All customization of the server can be done by editing the file
"syncserver.ini", which contains lots of comments to help you on
your way. Things you might like to change include:
* The client-visible hostname for your server. Edit the "public_url"
key under the [syncstorage] section.
* The database in which to store sync data. Edit the "sqluri" setting
under the [syncstorage] section.
Questions, Feedback
-------------------
- IRC channel: #sync. See http://irc.mozilla.org/
- Mailing list: https://mail.mozilla.org/listinfo/services-dev

13
sources/requirements.txt Normal file
View File

@@ -0,0 +1,13 @@
cornice==0.16.2
gunicorn==19.1.1
pyramid==1.5
requests==2.2.1
simplejson==3.4
SQLAlchemy==0.9.4
unittest2==0.5.1
https://github.com/mozilla-services/mozservices/archive/e00e1b68130423ad98d0f6185655bde650443da8.zip
https://github.com/mozilla-services/tokenserver/archive/d7e513e8a4f5c588b70d685a8df1d2e508c341c0.zip
http://github.com/mozilla-services/server-syncstorage/archive/1.5.5.zip
# Newer releases of configparser have b/w compat bug:
# https://github.com/mozilla-services/syncserver/issues/39
configparser==3.3.0r2

16
sources/setup.py Normal file
View File

@@ -0,0 +1,16 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from setuptools import setup
entry_points = """
[paste.app_factory]
main = syncserver:main
"""
setup(
name='syncserver',
version="1.5.1",
packages=['syncserver'],
entry_points=entry_points
)

35
sources/syncserver.ini Normal file
View File

@@ -0,0 +1,35 @@
[server:main]
use = egg:Paste#http
host = 0.0.0.0
port = 5000
[app:main]
use = egg:syncserver
[syncserver]
# This must be edited to point to the public URL of your server,
# i.e. the URL as seen by Firefox.
public_url = http://localhost:5000/
# This defines the database in which to store all server data.
#sqluri = sqlite:////tmp/syncserver.db
# This is a secret key used for signing authentication tokens.
# It should be long and randomly-generated.
# The following command will give a suitable value on *nix systems:
#
# head -c 20 /dev/urandom | sha1sum
#
# If not specified then the server will generate a temporary one at startup.
#secret = INSERT_SECRET_KEY_HERE
# Set this to "false" to disable new-user signups on the server.
# Only request by existing accounts will be honoured.
# allow_new_users = false
# Uncomment and edit the following to use a local BrowserID verifier
# rather than posing assertions to the mozilla-hosted verifier.
# Audiences should be set to your public_url without a trailing slash.
#[browserid]
#backend = tokenserver.verifiers.LocalVerifier
#audiences = https://localhost:5000

40
sources/syncserver.wsgi Normal file
View File

@@ -0,0 +1,40 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import sys
import site
from logging.config import fileConfig
from ConfigParser import NoSectionError
# detecting if virtualenv was used in this dir
_CURDIR = os.path.dirname(os.path.abspath(__file__))
_PY_VER = sys.version.split()[0][:3]
_SITE_PKG = os.path.join(_CURDIR, 'local', 'lib', 'python' + _PY_VER, 'site-packages')
# adding virtualenv's site-package and ordering paths
saved = sys.path[:]
if os.path.exists(_SITE_PKG):
site.addsitedir(_SITE_PKG)
for path in sys.path:
if path not in saved:
saved.insert(0, path)
sys.path[:] = saved
# setting up the egg cache to a place where apache can write
os.environ['PYTHON_EGG_CACHE'] = '/tmp/python-eggs'
# setting up logging
ini_file = os.path.join(_CURDIR, 'syncserver.ini')
try:
fileConfig(ini_file)
except NoSectionError:
pass
# running the app using Paste
from paste.deploy import loadapp
application = loadapp('config:%s'% ini_file)

146
sources/syncserver/__init__.py Normal file
View File

@@ -0,0 +1,146 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import logging
from urlparse import urlparse, urlunparse
from pyramid.response import Response
from pyramid.events import NewRequest, subscriber
import mozsvc.config
from tokenserver.util import _JSONError
logger = logging.getLogger("syncserver")
def includeme(config):
"""Install SyncServer application into the given Pyramid configurator."""
# Set the umask so that files are created with secure permissions.
# Necessary for e.g. created-on-demand sqlite database files.
os.umask(0077)
# Sanity-check the deployment settings and provide sensible defaults.
settings = config.registry.settings
public_url = settings.get("syncserver.public_url")
if public_url is None:
raise RuntimeError("you much configure syncserver.public_url")
public_url = public_url.rstrip("/")
settings["syncserver.public_url"] = public_url
secret = settings.get("syncserver.secret")
if secret is None:
secret = os.urandom(32).encode("hex")
sqluri = settings.get("syncserver.sqluri")
if sqluri is None:
rootdir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
sqluri = "sqlite:///" + os.path.join(rootdir, "syncserver.db")
# Configure app-specific defaults based on top-level configuration.
settings.pop("config", None)
if "tokenserver.backend" not in settings:
# Default to our simple static node-assignment backend
settings["tokenserver.backend"] =\
"syncserver.staticnode.StaticNodeAssignment"
settings["tokenserver.sqluri"] = sqluri
settings["tokenserver.node_url"] = public_url
settings["endpoints.sync-1.5"] = "{node}/storage/1.5/{uid}"
if "tokenserver.monkey_patch_gevent" not in settings:
# Default to no gevent monkey-patching
settings["tokenserver.monkey_patch_gevent"] = False
if "tokenserver.applications" not in settings:
# Default to just the sync-1.5 application
settings["tokenserver.applications"] = "sync-1.5"
if "tokenserver.secrets.backend" not in settings:
# Default to a single fixed signing secret
settings["tokenserver.secrets.backend"] = "mozsvc.secrets.FixedSecrets"
settings["tokenserver.secrets.secrets"] = [secret]
if "tokenserver.allow_new_users" not in settings:
allow_new_users = settings.get("syncserver.allow_new_users")
if allow_new_users is not None:
settings["tokenserver.allow_new_users"] = allow_new_users
if "hawkauth.secrets.backend" not in settings:
# Default to the same secrets backend as the tokenserver
for key in settings.keys():
if key.startswith("tokenserver.secrets."):
newkey = "hawkauth" + key[len("tokenserver"):]
settings[newkey] = settings[key]
if "storage.backend" not in settings:
# Default to sql syncstorage backend
settings["storage.backend"] = "syncstorage.storage.sql.SQLStorage"
settings["storage.sqluri"] = sqluri
settings["storage.create_tables"] = True
if "browserid.backend" not in settings:
# Default to remote verifier, with base of public_url as only audience
audience = urlunparse(urlparse(public_url)._replace(path=""))
settings["browserid.backend"] = "tokenserver.verifiers.RemoteVerifier"
settings["browserid.audiences"] = audience
if "loggers" not in settings:
# Default to basic logging config.
root_logger = logging.getLogger("")
if not root_logger.handlers:
logging.basicConfig(level=logging.INFO)
# Include the relevant sub-packages.
config.scan("syncserver")
config.include("syncstorage", route_prefix="/storage")
config.include("tokenserver", route_prefix="/token")
# Add a top-level "it works!" view.
def itworks(request):
return Response("it works!")
config.add_route('itworks', '/')
config.add_view(itworks, route_name='itworks')
@subscriber(NewRequest)
def reconcile_wsgi_environ_with_public_url(event):
"""Event-listener that checks and tweaks WSGI environ based on public_url.
This is a simple trick to help ensure that the configured public_url
matches the actual deployed address. It fixes fixes parts of the WSGI
environ where it makes sense (e.g. SCRIPT_NAME) and warns about any parts
that seem obviously mis-configured (e.g. http:// versus https://).
It's very important to get public_url and WSGI environ matching exactly,
since they're used for browserid audience checking and HAWK signature
validation, so mismatches can easily cause strange and cryptic errors.
"""
request = event.request
public_url = request.registry.settings["syncserver.public_url"]
p_public_url = urlparse(public_url)
# If we don't have a SCRIPT_NAME, take it from the public_url.
# This is often the case if we're behind e.g. an nginx proxy that
# is serving us at some sub-path.
if not request.script_name:
request.script_name = p_public_url.path.rstrip("/")
# Log a noisy error if the application url is different to what we'd
# expect based on public_url setting.
application_url = request.application_url
if public_url != application_url:
msg = "The public_url setting does not match the application url.\n"
msg += "This will almost certainly cause authentication failures!\n"
msg += " public_url setting is: %s\n" % (public_url,)
msg += " application url is: %s\n" % (application_url,)
logger.error(msg)
raise _JSONError([msg], status_code=500)
def get_configurator(global_config, **settings):
"""Load a SyncStorge configurator object from deployment settings."""
config = mozsvc.config.get_configurator(global_config, **settings)
config.begin()
try:
config.include(includeme)
finally:
config.end()
return config
def main(global_config, **settings):
"""Load a SyncStorage WSGI app from deployment settings."""
config = get_configurator(global_config, **settings)
return config.make_wsgi_app()

218
sources/syncserver/staticnode.py Normal file
View File

@@ -0,0 +1,218 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Simple node-assignment backend using a single, static node.
This is a greatly-simplified node-assignment backend. It keeps user records
in an SQL database, but does not attempt to do any node management. All users
are implicitly assigned to a single, static node.
XXX TODO: move this into the tokenserver repo.
"""
import time
import urlparse
from mozsvc.exceptions import BackendError
from sqlalchemy import Column, Integer, String, BigInteger, Index
from sqlalchemy import create_engine, Table, MetaData
from sqlalchemy.pool import QueuePool
from sqlalchemy.sql import text as sqltext
from sqlalchemy.exc import IntegrityError
from tokenserver.assignment import INodeAssignment
from zope.interface import implements
metadata = MetaData()
users = Table(
"users",
metadata,
Column("uid", Integer(), primary_key=True, autoincrement=True,
nullable=False),
Column("service", String(32), nullable=False),
Column("email", String(255), nullable=False),
Column("generation", BigInteger(), nullable=False),
Column("client_state", String(32), nullable=False),
Column("created_at", BigInteger(), nullable=False),
Column("replaced_at", BigInteger(), nullable=True),
Index('lookup_idx', 'email', 'service', 'created_at'),
Index('clientstate_idx', 'email', 'service', 'client_state', unique=True),
)
_GET_USER_RECORDS = sqltext("""\
select
uid, generation, client_state
from
users
where
email = :email
and
service = :service
order by
created_at desc, uid desc
limit
20
""")
_CREATE_USER_RECORD = sqltext("""\
insert into
users
(service, email, generation, client_state, created_at, replaced_at)
values
(:service, :email, :generation, :client_state, :timestamp, NULL)
""")
_UPDATE_GENERATION_NUMBER = sqltext("""\
update
users
set
generation = :generation
where
service = :service and email = :email and
generation < :generation and replaced_at is null
""")
_REPLACE_USER_RECORDS = sqltext("""\
update
users
set
replaced_at = :timestamp
where
service = :service and email = :email
and replaced_at is null and created_at < :timestamp
""")
def get_timestamp():
"""Get current timestamp in milliseconds."""
return int(time.time() * 1000)
class StaticNodeAssignment(object):
implements(INodeAssignment)
def __init__(self, sqluri, node_url, **kw):
self.sqluri = sqluri
self.node_url = node_url
self.driver = urlparse.urlparse(sqluri).scheme.lower()
sqlkw = {
"logging_name": "syncserver",
"connect_args": {},
"poolclass": QueuePool,
"pool_reset_on_return": True,
}
if self.driver == "sqlite":
# We must mark it as safe to share sqlite connections between
# threads. The pool will ensure there's on race conditions.
sqlkw["connect_args"]["check_same_thread"] = False
# If using a :memory: database, we must use a QueuePool of size
# 1 so that a single connection is shared by all threads.
if urlparse.urlparse(sqluri).path.lower() in ("/", "/:memory:"):
sqlkw["pool_size"] = 1
sqlkw["max_overflow"] = 0
self._engine = create_engine(sqluri, **sqlkw)
users.create(self._engine, checkfirst=True)
def get_user(self, service, email):
params = {'service': service, 'email': email}
res = self._engine.execute(_GET_USER_RECORDS, **params)
try:
row = res.fetchone()
if row is None:
return None
# The first row is the most up-to-date user record.
user = {
'email': email,
'uid': row.uid,
'node': self.node_url,
'generation': row.generation,
'client_state': row.client_state,
'old_client_states': {}
}
# Any subsequent rows are due to old client-state values.
row = res.fetchone()
while row is not None:
user['old_client_states'][row.client_state] = True
row = res.fetchone()
return user
finally:
res.close()
def allocate_user(self, service, email, generation=0, client_state=''):
params = {
'service': service, 'email': email, 'generation': generation,
'client_state': client_state, 'timestamp': get_timestamp()
}
try:
res = self._engine.execute(_CREATE_USER_RECORD, **params)
except IntegrityError:
raise
return self.get_user(service, email)
else:
res.close()
return {
'email': email,
'uid': res.lastrowid,
'node': self.node_url,
'generation': generation,
'client_state': client_state,
'old_client_states': {}
}
def update_user(self, service, user, generation=None, client_state=None):
if client_state is None:
# uid can stay the same, just update the generation number.
if generation is not None:
params = {
'service': service,
'email': user['email'],
'generation': generation,
}
res = self._engine.execute(_UPDATE_GENERATION_NUMBER, **params)
res.close()
user['generation'] = max(generation, user['generation'])
else:
# reject previously-seen client-state strings.
if client_state == user['client_state']:
raise BackendError('previously seen client-state string')
if client_state in user['old_client_states']:
raise BackendError('previously seen client-state string')
# need to create a new record for new client_state.
if generation is not None:
generation = max(user['generation'], generation)
else:
generation = user['generation']
now = get_timestamp()
params = {
'service': service, 'email': user['email'],
'generation': generation, 'client_state': client_state,
'timestamp': now,
}
try:
res = self._engine.execute(_CREATE_USER_RECORD, **params)
except IntegrityError:
user.update(self.get_user(service, user['email']))
else:
self.get_user(service, user['email'])
user['uid'] = res.lastrowid
user['generation'] = generation
user['old_client_states'][user['client_state']] = True
user['client_state'] = client_state
res.close()
# mark old records as having been replaced.
# if we crash here, they are unmarked and we may fail to
# garbage collect them for a while, but the active state
# will be undamaged.
params = {
'service': service, 'email': user['email'], 'timestamp': now
}
res = self._engine.execute(_REPLACE_USER_RECORDS, **params)
res.close()

17
sources/syncserver/tests.ini Normal file
View File

@@ -0,0 +1,17 @@
[server:main]
use = egg:Paste#http
host = 0.0.0.0
port = 5000
[app:main]
use = egg:SyncServer
[syncserver]
# This must be edited to point to the public URL of your server.
public_url = http://localhost:5000/
# This defines the database in which to store all server data.
#sqluri = sqlite:////tmp/syncserver.db
# This is a secret key used for signing authentication tokens.
#secret = INSERT_SECRET_KEY_HERE