Merge pull request #6 from Jibec/master

Use new helpers, add backup/restore and uwsgi
master
Josue-T 6 years ago committed by GitHub
commit f021251f30
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -1,21 +1,18 @@
# Mozillas Sync Server for Yunohost
The Sync Server provides a replacement for Firefoxs default server (hosted at Mozilla). Its code is available on [the Sync-1.5 Server documentation](https://docs.services.mozilla.com/howtos/run-sync-1.5.html).
The Sync Server provides a replacement for Firefoxs default server (hosted at Mozilla).
By default, a server set up will defer authentication to the Mozilla-hosted accounts server at [https://accounts.firefox.com](https://accounts.firefox.com). So you will still have to authenticate at Mozilla, but _the storage of your information will be done on your host_.
**Shipped version:** 1.5
**Shipped version:** 1.8
## Configuring
Once installed, reaching `http://domain.tld/path` should show a page explaining how to configure it. Otherwise please refer to the [Yunohost page](https://yunohost.org/#/app_ffsync).
## Contributing
## Links
From the `sources` directory, do as follows:
`make build`
`make test`
`./local/bin/gunicorn --reload --paste syncserver.ini` instead of the classical `make serve` to take into account changes you will surely do by developping the app.
* Report a bug about this package: https://github.com/YunoHost-Apps/ffsync_ynh
* Report a bug about Funkwhale itself: https://github.com/mozilla-services/syncserver
* Documentation: https://docs.services.mozilla.com/howtos/run-sync-1.5.html
* YunoHost website: https://yunohost.org/

@ -0,0 +1,41 @@
;; Test name
; Manifest
domain="domain.tld" (DOMAIN)
path="/path" (PATH)
; Checks
pkg_linter=1
setup_sub_dir=1
setup_root=1
setup_nourl=0
setup_private=0
setup_public=1
upgrade=1
upgrade=1 from_commit=fd6350495d5a1d864ae30e1a61e18939fdb6a428
upgrade=1 from_commit=267ccc21f7b52d22bc3d5b9cd6239857b9a82aad
backup_restore=1
multi_instance=1
incorrect_path=1
port_already_use=0
change_url=0
;;; Levels
Level 1=auto
Level 2=auto
Level 3=auto
# impossible with Firefox Sync (it uses Firefox Accounts)
Level 4=na
Level 5=auto
Level 6=auto
Level 7=auto
Level 8=0
Level 9=0
Level 10=0
;;; Options
Email=jean-baptiste@holcroft.fr
Notification=fail
;;; Upgrade options
; commit=fd6350495d5a1d864ae30e1a61e18939fdb6a428
name=latest published version
manifest_arg=domain=DOMAIN&path=PATH
; commit=267ccc21f7b52d22bc3d5b9cd6239857b9a82aad
name=latest git commit before Jibec's rewriting
manifest_arg=domain=DOMAIN&path=PATH

@ -0,0 +1,4 @@
SOURCE_URL=https://github.com/mozilla-services/syncserver/archive/1.8.0.tar.gz
SOURCE_SUM=728206bcffec7a305e97e7cd4d465b3fa56f39f8e1fd55d98e49a866016d61e0
SOURCE_SUM_PRG=sha256sum
SOURCE_FORMAT=tar.gz

@ -1,75 +0,0 @@
#!/bin/bash
# /etc/init.d/sync
# version 0.1 2013-03-12 (YYYY-MM-DD)
### BEGIN INIT INFO
# Provides: sync
# Required-Start: $local_fs $remote_fs
# Required-Stop: $local_fs $remote_fs
# Should-Start: $network
# Should-Stop: $network
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: Mozilla Sync server
# Description: Starts the mozilla sync server
### END INIT INFO
# Source function library.
#. /etc/rc.d/init.d/functions
prog=sync
SYNC_USER=ffsync
SYNC_HOME=/opt/yunohost/ffsync
CPU_COUNT=2
pidfile=/tmp/sync.pid
lockfile=/var/run/sync.lock
conffile=${SYNC_HOME}/syncserver.ini
GUNICORN=${SYNC_HOME}/local/bin/gunicorn
GUNICORN_ARGS="--paste $conffile --access-logfile /var/log/ffsync.log --daemon -p $pidfile"
start () {
echo -n "Starting $prog"
start-stop-daemon --start -c ${SYNC_USER} --exec $GUNICORN -- $GUNICORN_ARGS
RETVAL=$?
echo
[ $RETVAL = 0 ] && touch ${lockfile}
return $RETVAL
}
stop() {
echo "Stopping $prog"
start-stop-daemon --stop --quiet --oknodo --pidfile ${pidfile}
#log_end_msg $?
rm -f ${pidfile}
}
status(){
if [[ -f ${pidfile} ]]; then
echo "Status: running."
exit 0;
else
echo "Status: not running."
exit 1;
fi
}
case "$1" in
start)
start
;;
stop)
stop
;;
status)
status
;;
restart)
stop
start
;;
*)
echo $"Usage: $prog {start|stop|restart|help}"
RETVAL=2
esac
exit $RETVAL

@ -1,9 +0,0 @@
"/var/log/ffsync.log" {
copytruncate
daily
rotate 7
compress
delaycompress
missingok
notifempty
}

@ -1,15 +1,17 @@
location PATHTOCHANGE {
#sub_path_only rewrite ^__PATH__$ __PATH__/ permanent;
location __PATH__/ {
# Path to source
alias __FINALPATH__/ ;
if ($scheme = http) {
rewrite ^ https://$server_name$request_uri? permanent;
}
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Real-IP $remote_addr;
proxy_redirect off;
proxy_read_timeout 120;
proxy_connect_timeout 10;
proxy_pass http://127.0.0.1:5000/;
include conf.d/yunohost_panel.conf.inc;
include uwsgi_params;
# Needed for long running operations in admin interface
uwsgi_read_timeout 3600;
uwsgi_param SCRIPT_NAME __PATH__;
uwsgi_modifier1 30;
uwsgi_pass unix:///var/run/__NAME__/app.socket;
}

@ -1,45 +0,0 @@
[server:main]
use = egg:gunicorn
host = 0.0.0.0
port = 5000
workers = 1
timeout = 30
[app:main]
use = egg:syncserver
[syncserver]
# This must be edited to point to the public URL of your server,
# i.e. the URL as seen by Firefox.
public_url = https://ynhbaseurl/
# This defines the database in which to store all server data.
sqluri = pymysql://yunouser:yunopass@localhost/yunobase
# This is a secret key used for signing authentication tokens.
# It should be long and randomly-generated.
# The following command will give a suitable value on *nix systems:
#
# head -c 20 /dev/urandom | sha1sum
#
# If not specified then the server will generate a temporary one at startup.
secret = changesecret
# Set this to "false" to disable new-user signups on the server.
# Only request by existing accounts will be honoured.
allow_new_users = true
# Set this to "true" to work around a mismatch between public_url and
# the application URL as seen by python, which can happen in certain reverse-
# proxy hosting setups. It will overwrite the WSGI environ dict with the
# details from public_url. This could have security implications if e.g.
# you tell the app that it's on HTTPS but it's really on HTTP, so it should
# only be used as a last resort and after careful checking of server config.
force_wsgi_environ = true
# Uncomment and edit the following to use a local BrowserID verifier
# rather than posting assertions to the mozilla-hosted verifier.
# Audiences should be set to your public_url without a trailing slash.
#[browserid]
#backend = tokenserver.verifiers.LocalVerifier
#audiences = https://localhost:5000

@ -0,0 +1,20 @@
[Unit]
Description=%i uWSGI app
After=syslog.target
[Service]
RuntimeDirectory=%i
ExecStart=/usr/bin/uwsgi \
--ini /etc/uwsgi/apps-available/%i.ini \
--socket /var/run/%i/app.socket \
--logto /var/log/uwsgi/%i/%i.log
User=%i
Group=www-data
Restart=on-failure
KillSignal=SIGQUIT
Type=notify
StandardError=syslog
NotifyAccess=all
[Install]
WantedBy=multi-user.target

@ -1,9 +1,14 @@
[server:main]
use = egg:gunicorn
host = 0.0.0.0
port = 5000
workers = 1
timeout = 30
[uwsgi]
plugins = python
master = true
protocol = uwsgi
socket = /var/run/__APP__/app.socket
chmod-socket = 660
virtualenv = __FINALPATH__/local
wsgi-file = __FINALPATH__/syncserver.wsgi
python-path = __FINALPATH__/local
enable-threads = true
close-on-exec = true
[app:main]
use = egg:syncserver
@ -11,10 +16,10 @@ use = egg:syncserver
[syncserver]
# This must be edited to point to the public URL of your server,
# i.e. the URL as seen by Firefox.
public_url = http://localhost:5000/
public_url = https://__DOMAIN____PATH__
# This defines the database in which to store all server data.
#sqluri = sqlite:////tmp/syncserver.db
sqluri = pymysql://__DB_USER__:__DB_PWD__@localhost/__DB_NAME__
# This is a secret key used for signing authentication tokens.
# It should be long and randomly-generated.
@ -23,11 +28,11 @@ public_url = http://localhost:5000/
# head -c 20 /dev/urandom | sha1sum
#
# If not specified then the server will generate a temporary one at startup.
#secret = INSERT_SECRET_KEY_HERE
secret = __SECRET__
# Set this to "false" to disable new-user signups on the server.
# Only request by existing accounts will be honoured.
# allow_new_users = false
allow_new_users = true
# Set this to "true" to work around a mismatch between public_url and
# the application URL as seen by python, which can happen in certain reverse-
@ -42,4 +47,3 @@ force_wsgi_environ = false
# Audiences should be set to your public_url without a trailing slash.
#[browserid]
#backend = tokenserver.verifiers.LocalVerifier
#audiences = https://localhost:5000

@ -1,25 +1,31 @@
{
"name": "Firefox Sync Server",
"id": "ffsync",
"url": "https://github.com/abeudin/ffsync_ynh",
"version": "1.8.0~ynh1",
"url": "https://github.com/mozilla-services/syncserver",
"packaging_format": 1,
"description": {
"en": "Mozillas Sync-Server to host your Firefox account data",
"fr": "Le serveur de synchronisation de Mozilla, pour héberger vos données Firefox"
},
"maintainer": {
"name": "Josué Tille",
"email": "josue@tille.ch"
},
"previous_maintainers": [{
"name": "beudbeud",
"email": "beudbeud@beudibox.fr",
"url": "https://github.com/balu-/FSyncMS"
},
"name": "jibec",
"email": "jean-baptiste@holcroft.fr"
}],
"requirements": {
"yunohost": ">> 2.4.0"
"yunohost": ">=3.0.0"
},
"license": "free",
"license": "MPL-2.0",
"services": [
"nginx"
],
"multi_instance": false,
"multi_instance": true,
"arguments": {
"install" : [
{

@ -0,0 +1,95 @@
#!/bin/bash
# Check if system wide templates are available and correcly configured
#
# usage: ynh_check_global_uwsgi_config
ynh_check_global_uwsgi_config () {
uwsgi --version || ynh_die "You need to add uwsgi (and appropriate plugin) as a dependency"
cp ../conf/uwsgi-app@.service /etc/systemd/system/uwsgi-app@.service
systemctl daemon-reload
}
# Create a dedicated uwsgi ini file to use with generic uwsgi service
#
# This will use a template in ../conf/uwsgi.ini
# and will replace the following keywords with
# global variables that should be defined before calling
# this helper :
#
# __APP__ by $app
# __PATH__ by $path_url
# __FINALPATH__ by $final_path
#
# And dynamic variables (from the last example) :
# __PATH_2__ by $path_2
# __PORT_2__ by $port_2
#
# usage: ynh_add_uwsgi_service
#
# to interact with your service: `systemctl <action> uwsgi-app@app`
ynh_add_uwsgi_service () {
ynh_check_global_uwsgi_config
local others_var=${1:-}
local finaluwsgiini="/etc/uwsgi/apps-available/$app.ini"
# www-data group is needed since it is this nginx who will start the service
usermod --append --groups www-data "$app" || ynh_die "It wasn't possible to add user $app to group www-data"
ynh_backup_if_checksum_is_different "$finaluwsgiini"
cp ../conf/uwsgi.ini "$finaluwsgiini"
# To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable.
# Substitute in a nginx config file only if the variable is not empty
if test -n "${final_path:-}"; then
ynh_replace_string "__FINALPATH__" "$final_path" "$finaluwsgiini"
fi
if test -n "${path_url:-}"; then
ynh_replace_string "__PATH__" "$path_url" "$finaluwsgiini"
fi
if test -n "${app:-}"; then
ynh_replace_string "__APP__" "$app" "$finaluwsgiini"
fi
# Replace all other variable given as arguments
for var_to_replace in $others_var
do
# ${var_to_replace^^} make the content of the variable on upper-cases
# ${!var_to_replace} get the content of the variable named $var_to_replace
ynh_replace_string "__${var_to_replace^^}__" "${!var_to_replace}" "$finaluwsgiini"
done
ynh_store_file_checksum "$finaluwsgiini"
chown $app:root "$finaluwsgiini"
# make sure the folder for logs exists and set authorizations
mkdir -p /var/log/uwsgi/$app
chown $app:root /var/log/uwsgi/$app
chmod -R u=rwX,g=rX,o= /var/log/uwsgi/$app
systemctl daemon-reload
systemctl stop "uwsgi-app@$app.service"
systemctl enable "uwsgi-app@$app.service"
systemctl start "uwsgi-app@$app.service"
# Add as a service
yunohost service add "uwsgi-app@$app" --log "/var/log/uwsgi/$app/$app.log"
}
# Remove the dedicated uwsgi ini file
#
# usage: ynh_remove_uwsgi_service
ynh_remove_uwsgi_service () {
local finaluwsgiini="/etc/uwsgi/apps-available/$app.ini"
if [ -e "$finaluwsgiini" ]; then
systemctl stop "uwsgi-app@$app.service"
systemctl disable "uwsgi-app@$app.service"
yunohost service remove "uwsgi-app@$app"
ynh_secure_remove "$finaluwsgiini"
ynh_secure_remove "/var/log/uwsgi/$app"
fi
}

@ -0,0 +1,62 @@
#!/bin/bash
#=================================================
# GENERIC START
#=================================================
# IMPORT GENERIC HELPERS
#=================================================
source ../settings/scripts/_common.sh
source /usr/share/yunohost/helpers
#=================================================
# MANAGE SCRIPT FAILURE
#=================================================
# Exit if an error occurs during the execution of the script
ynh_abort_if_errors
#=================================================
# LOAD SETTINGS
#=================================================
app=$YNH_APP_INSTANCE_NAME
final_path=$(ynh_app_setting_get "$app" final_path)
domain=$(ynh_app_setting_get "$app" domain)
db_name=$(ynh_app_setting_get "$app" db_name)
#=================================================
# STANDARD BACKUP STEPS
#=================================================
# BACKUP THE APP MAIN DIR
#=================================================
ynh_backup "$final_path"
#=================================================
# BACKUP THE NGINX CONFIGURATION
#=================================================
ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf"
#=================================================
# BACKUP THE MYSQL DATABASE
#=================================================
ynh_mysql_dump_db "$db_name" > db.sql
#=================================================
# SPECIFIC BACKUP
#=================================================
# Backup Log
#=================================================
ynh_backup "/var/log/uwsgi/$app"
#=================================================
# BACKUP THE UWSGI FILES
#=================================================
ynh_backup "/etc/uwsgi/apps-available/$app.ini"
ynh_backup "/etc/systemd/system/uwsgi-app@.service"

@ -1,105 +1,177 @@
#!/bin/bash
# Source app helpers
. /usr/share/yunohost/helpers
#=================================================
# GENERIC START
#=================================================
# IMPORT GENERIC HELPERS
#=================================================
source _common.sh
source /usr/share/yunohost/helpers
#=================================================
# MANAGE SCRIPT FAILURE
#=================================================
# Exit if an error occurs during the execution of the script
ynh_abort_if_errors
#=================================================
# RETRIEVE ARGUMENTS FROM THE MANIFEST
#=================================================
# Retrieve arguments
domain=$1
path=$2
domain=$YNH_APP_ARG_DOMAIN
path_url=$YNH_APP_ARG_PATH
# Check domain/path availability
sudo yunohost app checkurl $domain$path -a ffsync
if [[ ! $? -eq 0 ]]; then
ynh_die
fi
app=$YNH_APP_INSTANCE_NAME
final_path="/opt/yunohost/$app"
# Generate random password
db_pwd=$(head -c 8 /dev/urandom | sha1sum | cut -d " " -f1)
# Normalize the url path syntax
path_url=$(ynh_normalize_url_path "$path_url")
# Use 'FSyncMS' as database name and user
db_user=ffsync
#=================================================
# STORE SETTINGS FROM MANIFEST
#=================================================
# Initialize database and store mysql password for upgrade
sudo yunohost app initdb $db_user -p $db_pwd
ynh_app_setting_set ffsync mysqlpwd $db_pwd
ynh_app_setting_set "$app" final_path "$final_path"
#=================================================
# CHECK IF THE APP CAN BE INSTALLED WITH THESE ARGS
#=================================================
# Check destination directory
test ! -e "$final_path" || ynh_die "This path already contains a folder"
# Check web path availability
ynh_webpath_available "$domain" "$path_url"
# Register (book) web path
ynh_webpath_register "$app" "$domain" "$path_url"
#=================================================
# STANDARD MODIFICATIONS
#=================================================
# Generate random password and save
secret=$(head -c 20 /dev/urandom | sha1sum | cut -d " " -f1)
ynh_app_setting_set ffsync secret $secret
#=================================================
# INSTALL DEPENDENCIES
#=================================================
# Check depends installation
sudo apt-get install make python-dev python-virtualenv -y
ynh_install_app_dependencies python-dev python-virtualenv \
uwsgi uwsgi-plugin-python \
`# ARM support: ` \
build-essential libssl-dev libffi-dev
# Check Swap
if [ $(sudo swapon -s | wc -l) = 1 ];
then
# It is NOT possible to setup a swap file on a tmpfs filesystem
mount | grep /tmp | grep tmpfs > /dev/null 2>&1
if [ $? = 1 ];
then
tmp_swap_file=/tmp/ffsync_swapfile
else
tmp_swap_file=/var/cache/ffsync_swapfile
fi
sudo dd if=/dev/zero of=$tmp_swap_file bs=1M count=256
sudo chmod 600 $tmp_swap_file
sudo mkswap $tmp_swap_file
sudo swapon $tmp_swap_file
fi
#=================================================
# CREATE A MYSQL DATABASE
#=================================================
# Use 'FSyncMS' as database name and user
db_user=$app
db_name=$(ynh_sanitize_dbid $app)
# Modify assets to take path into account
sudo find ../sources/syncserver/page/sync_files/ -type f -exec sed -i -e "s@media\/img@$path\/media\/img@g" {} \;
ynh_app_setting_set "$app" db_name "$db_name"
ynh_mysql_setup_db "$db_user" "$db_name"
# Copy files to the right place
final_path=/opt/yunohost/ffsync
sudo mkdir -p $final_path
sudo cp -a ../sources/* $final_path
sudo cp ../conf/ffsync /etc/init.d/
sudo cp ../conf/ffsync.logrotate /etc/logrotate.d/ffsync
sudo touch /var/log/ffsync.log
#=================================================
# DOWNLOAD, CHECK AND UNPACK SOURCE
#=================================================
# Set permissions to ffsync directory
sudo useradd ffsync -d $final_path
sudo chown ffsync:ffsync -R $final_path
sudo chown ffsync /var/log/ffsync.log
# Download, check integrity, uncompress and patch the source from app.src
ynh_setup_source "$final_path"
# Modify Nginx configuration file and copy it to Nginx conf directory
sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf
sed -i "s@ALIASTOCHANGE@$final_path/@g" ../conf/nginx.conf
sudo cp ../conf/nginx.conf /etc/nginx/conf.d/$domain.d/ffsync.conf
sudo cp ../conf/syncserver.ini $final_path/syncserver.ini
sudo sed -i -e "s@ynhbaseurl@$domain$path@g" $final_path/syncserver.ini
sudo sed -i -e "s@changesecret@$secret@g" $final_path/syncserver.ini
sudo sed -i "s/yunouser/$db_user/g" $final_path/syncserver.ini
sudo sed -i "s/yunopass/$db_pwd/g" $final_path/syncserver.ini
sudo sed -i "s/yunobase/$db_user/g" $final_path/syncserver.ini
sudo sed -i -e "s@media\/img@$path\/media\/img@g" $final_path/syncserver/page/sync_files/firefox_sync-bundle.css
sudo sed -i -e "s@media\/img@$path\/media\/img@g" $final_path/syncserver/page/sync_files/responsive-bundle.css
# Modify assets to take path into account
# TODO: try to include this as a patch if still needed
# find ../sources/syncserver/page/sync_files/ -type f -exec sed -i -e "s@media\/img@$path_url\/media\/img@g" {} \;
# Init virtualenv
cd $final_path && sudo make build && sudo ./local/bin/easy_install gunicorn
#=================================================
# NGINX CONFIGURATION
#=================================================
# Disable swapfile
if [ -z ${tmp_swap_file+x} ];
# Modify Nginx configuration file and copy it to Nginx conf directory
ynh_add_nginx_config
if [ "$path_url" == "/" ]
then
sudo swapoff $tmp_swap_file
sudo rm -f $tmp_swap_file
# $finalnginxconf comes from ynh_add_nginx_config
# uwsgi_param is only needed for non-root installation
ynh_replace_string "uwsgi_param " "#uwsgi_param " "$finalnginxconf"
ynh_replace_string "uwsgi_modifier1 " "#uwsgi_modifier1 " "$finalnginxconf"
fi
ynh_store_file_checksum "$finalnginxconf"
systemctl reload nginx
#=================================================
# CREATE DEDICATED USER
#=================================================
ynh_system_user_create "$app" "$final_path"
#=================================================
# SPECIFIC SETUP
#=================================================
# pip installation
#=================================================
virtualenv "$final_path/local"
# Init virtualenv
(
set +o nounset
source "$final_path/local/bin/activate"
set -o nounset
cd "$final_path"
pip install --upgrade pip
pip install pyramid_chameleon
CFLAGS="-Wno-error -Wno-error=format-security" \
ARCHFLAGS="-Wno-error=unused-command-line-argument-hard-error-in-future" \
pip install --requirement "$final_path/requirements.txt"
python "$final_path/setup.py" develop
touch "$final_path/local/COMPLETE"
)
# Add nice homepage
cp -r ../sources/page $final_path/syncserver/
(cd "$final_path/syncserver" && patch -p1 < $YNH_CWD/../sources/homepage.patch) || echo "Unable to apply patches"
#=================================================
# SECURE FILES AND DIRECTORIES
#=================================================
chown $app -R $final_path
chmod u=rwX,g=rX,o= -R $final_path
#=================================================
# SETUP UWSGI
#=================================================
# Generate random password and save
secret=$(ynh_string_random)
ynh_app_setting_set "$app" secret "$secret"
# create config file syncserver.ini
rm "$final_path/syncserver.ini"
ln -s "/etc/uwsgi/apps-available/$app.ini" "$final_path/syncserver.ini"
# configure uwsgi
ynh_add_uwsgi_service 'domain secret db_user db_pwd db_name'
#=================================================
# MODIFY A CONFIG FILE
#=================================================
# TODO: fix this css patch
# ynh_replace_string "media\/img@$path_url\/media\/img@g" $final_path/syncserver/page/sync_files/firefox_sync-bundle.css
# ynh_replace_string "media\/img@$path_url\/media\/img@g" $final_path/syncserver/page/sync_files/responsive-bundle.css
#=================================================
# GENERIC FINALIZATION
#=================================================
# SETUP SSOWAT
#=================================================
# Fix permission
sudo find $final_path/ -type d -exec chmod 2755 {} \;
sudo find $final_path/ -type f -exec chmod g+r,o+r {} \;
#enable services
sudo chmod +x /etc/init.d/ffsync
sudo update-rc.d ffsync defaults
sudo service ffsync restart
sudo service ffsync restart
sudo service ffsync restart
# Reload Nginx and regenerate SSOwat conf
sudo yunohost app ssowatconf
sudo service nginx restart
sudo yunohost service add ffsync -l /var/log/ffsync.log
ynh_app_setting_set ffsync skipped_uris "/"
# accessible by everyone (authentification is done by firefox accounts)
ynh_app_setting_set "$app" skipped_uris "/"

@ -1,22 +1,63 @@
#!/bin/bash
# Source app helpers
. /usr/share/yunohost/helpers
db_user=ffsync
db_name=ffsync
root_pwd=$(sudo cat /etc/yunohost/mysql)
domain=$(ynh_app_setting_get ffsync domain)
mysql -u root -p$root_pwd -e "DROP DATABASE $db_name ; DROP USER $db_user@localhost ;"
sudo rm -rf /opt/yunohost/ffsync
sudo rm -f /etc/nginx/conf.d/$domain.d/ffsync.conf
sudo service ffsync stop
sudo update-rc.d ffsync remove
sudo rm /etc/init.d/ffsync
sudo rm /etc/logrotate.d/ffsync
sudo yunohost service remove ffsync
sudo service nginx reload
sudo userdel ffsync
sudo delgroup ffsync
#=================================================
# GENERIC START
#=================================================
# IMPORT GENERIC HELPERS
#=================================================
source _common.sh
source /usr/share/yunohost/helpers
#=================================================
# LOAD SETTINGS
#=================================================
app=$YNH_APP_INSTANCE_NAME
domain=$(ynh_app_setting_get "$app" domain)
final_path=$(ynh_app_setting_get "$app" final_path)
db_user=$app
db_name=$app
#=================================================
# STANDARD REMOVE
#=================================================
# REMOVE UWSGI
#=================================================
ynh_remove_uwsgi_service
#=================================================
# REMOVE DEPENDENCIES
#=================================================
# Remove metapackage and its dependencies
ynh_remove_app_dependencies
#=================================================
# REMOVE THE MYSQL DATABASE
#=================================================
ynh_mysql_remove_db "$db_user" "$db_name"
#=================================================
# REMOVE APP MAIN DIR
#=================================================
ynh_secure_remove "$final_path"
#=================================================
# REMOVE NGINX CONFIGURATION
#=================================================
ynh_remove_nginx_config
#=================================================
# GENERIC FINALIZATION
#=================================================
# REMOVE DEDICATED USER
#=================================================
# Delete a system user
ynh_system_user_delete "$app"

@ -0,0 +1,105 @@
#!/bin/bash
#=================================================
# GENERIC START
#=================================================
# IMPORT GENERIC HELPERS
#=================================================
source ../settings/scripts/_common.sh
source /usr/share/yunohost/helpers
#=================================================
# MANAGE SCRIPT FAILURE
#=================================================
# Exit if an error occurs during the execution of the script
ynh_abort_if_errors
#=================================================
# LOAD SETTINGS
#=================================================
app=$YNH_APP_INSTANCE_NAME
domain=$(ynh_app_setting_get "$app" domain)
path_url=$(ynh_app_setting_get "$app" path)
final_path=$(ynh_app_setting_get "$app" final_path)
db_name=$(ynh_app_setting_get "$app" db_name)
#=================================================
# CHECK IF THE APP CAN BE RESTORED
#=================================================
ynh_webpath_available "$domain" "$path_url" \
|| ynh_die "Path not available: ${domain}${path_url}"
test ! -d "$final_path" \
|| ynh_die "There is already a directory: $final_path "
#=================================================
# STANDARD RESTORATION STEPS
#=================================================
# Restore all config and data
ynh_restore
#=================================================
# RESTORE THE MYSQL DATABASE
#=================================================
db_pwd=$(ynh_app_setting_get "$app" mysqlpwd)
ynh_mysql_setup_db "$db_name" "$db_name" "$db_pwd"
ynh_mysql_connect_as "$db_name" "$db_pwd" "$db_name" < ./db.sql
#=================================================
# RECREATE THE DEDICATED USER
#=================================================
# Create the dedicated user (if not existing)
ynh_system_user_create "$app"
#=================================================
# RESTORE USER RIGHTS
#=================================================
chown $app -R $final_path
chmod u=rwX,g=rX,o= -R $final_path
#=================================================
# SPECIFIC RESTORATION
#=================================================
# REINSTALL DEPENDENCIES
#=================================================
ynh_install_app_dependencies python-dev python-virtualenv \
uwsgi uwsgi-plugin-python \
`# ARM support: ` \
build-essential libssl-dev libffi-dev
# set authorizations
chown $app:root /var/log/uwsgi/$app
chmod -R u=rwX,g=rX,o= /var/log/uwsgi/$app
#=================================================
# RESTORE SERVICE
#=================================================
usermod --append --groups www-data "$app"
systemctl daemon-reload
systemctl enable "uwsgi-app@$app.service"
#=================================================
# ADVERTISE SERVICE IN ADMIN PANEL
#=================================================
yunohost service add "uwsgi-app@$app.service" --log "/var/log/uwsgi/app/$app"
#=================================================
# GENERIC FINALIZATION
#=================================================
# RELOAD NGINX AND PHP-FPM
#=================================================
systemctl start "uwsgi-app@$app.service"
systemctl reload nginx

@ -1,91 +1,182 @@
#!/bin/bash
# Source app helpers
. /usr/share/yunohost/helpers
#=================================================
# GENERIC START
#=================================================
# IMPORT GENERIC HELPERS
#=================================================
source _common.sh
source /usr/share/yunohost/helpers
#=================================================
# MANAGE SCRIPT FAILURE
#=================================================
# Exit if an error occurs during the execution of the script
ynh_abort_if_errors
#=================================================
# RETRIEVE ARGUMENTS FROM THE MANIFEST
#=================================================
app=$YNH_APP_INSTANCE_NAME
# Retrieve arguments
domain=$(ynh_app_setting_get ffsync domain)
path=$(ynh_app_setting_get ffsync path)
db_pwd=$(ynh_app_setting_get ffsync mysqlpwd)
db_user=ffsync
final_path=/opt/yunohost/ffsync
# Get secret variable
secret=$(ynh_app_setting_get ffsync secret)
# Get from conf file if not defined
if [[ -z $secret ]]
then
secret=$(sudo grep "secret =" $final_path/syncserver.ini | cut -d" " -f3)
ynh_app_setting_set ffsync secret $secret
domain=$(ynh_app_setting_get "$app" domain)
path_url=$(ynh_app_setting_get "$app" path)
db_name=$(ynh_app_setting_get "$app" db_name)
db_pwd=$(ynh_app_setting_get "$app" mysqlpwd)
db_user=$app
final_path=$(ynh_app_setting_get "$app" final_path)
secret=$(ynh_app_setting_get "$app" secret)
#=================================================
# ENSURE DOWNWARD COMPATIBILITY
#=================================================
# If db_name doesn't exist, create it
if [ -z "$db_name" ]; then
db_name=$(ynh_sanitize_dbid "$app")
ynh_app_setting_set $app db_name "$db_name"
fi
# Check Swap
if [ $(sudo swapon -s | wc -l) = 1 ];
then
# It is NOT possible to setup a swap file on a tmpfs filesystem
mount | grep /tmp | grep tmpfs > /dev/null 2>&1
if [ $? = 1 ];
then
tmp_swap_file=/tmp/ffsync_swapfile
else
tmp_swap_file=/var/cache/ffsync_swapfile
# If final_path doesn't exist, create it
if [ -z "$final_path" ]; then
final_path=/opt/yunohost/$app
ynh_app_setting_set "$app" final_path "$final_path"
fi
sudo dd if=/dev/zero of=$tmp_swap_file bs=1M count=256
sudo chmod 600 $tmp_swap_file
sudo mkswap $tmp_swap_file
sudo swapon $tmp_swap_file
# If path_url doesn't exist, create it
if [ -z "$path_url" ]; then
path_url=$(ynh_app_setting_get "$app" path)
ynh_app_setting_set "$app" path "$path_url"
ynh_app_setting_delete "$app" path
fi
# Copy files to the right place
sudo mkdir -p $final_path
sudo cp -a ../sources/* $final_path
sudo cp ../conf/ffsync /etc/init.d/
sudo cp ../conf/ffsync.logrotate /etc/logrotate.d/ffsync
# Set permissions to ffsync directory
sudo useradd ffsync -d $final_path
sudo chown ffsync:ffsync -R $final_path
# Modify Nginx configuration file and copy it to Nginx conf directory
sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf
sed -i "s@ALIASTOCHANGE@$final_path@g" ../conf/nginx.conf
sudo cp ../conf/nginx.conf /etc/nginx/conf.d/$domain.d/ffsync.conf
sudo cp ../conf/syncserver.ini $final_path/syncserver.ini
sudo sed -i -e "s@ynhbaseurl@$domain$path@g" $final_path/syncserver.ini
sudo sed -i -e "s@changesecret@$secret@g" $final_path/syncserver.ini
sudo sed -i "s/yunouser/$db_user/g" $final_path/syncserver.ini
sudo sed -i "s/yunopass/$db_pwd/g" $final_path/syncserver.ini
sudo sed -i "s/yunobase/$db_user/g" $final_path/syncserver.ini
sudo sed -i -e "s@media\/img@$path\/media\/img@g" $final_path/syncserver/page/sync_files/firefox_sync-bundle.css
sudo sed -i -e "s@media\/img@$path\/media\/img@g" $final_path/syncserver/page/sync_files/responsive-bundle.css
# stop service before upgrade
sudo service ffsync stop
# Detect old installation style
if [ -e /etc/init.d/ffsync ]; then
service ffsync stop
update-rc.d -f ffsync remove
# Init virtualenv
cd $final_path && sudo make build && sudo ./local/bin/easy_install gunicorn
ynh_secure_remove /etc/init.d/ffsync
ynh_secure_remove /var/log/ffsync.log
ynh_secure_remove /opt/yunohost/ffsync
yunohost service remove "$app"
fi
#=================================================
# BACKUP BEFORE UPGRADE THEN ACTIVE TRAP
#=================================================
# Backup the current version of the app
ynh_backup_before_upgrade
ynh_clean_setup () {
# restore it if the upgrade fails
ynh_restore_upgradebackup
}
# Exit if an error occurs during the execution of the script
ynh_abort_if_errors
#=================================================
# CHECK THE PATH
#=================================================
# Normalize the URL path syntax
path_url=$(ynh_normalize_url_path "$path_url")
#=================================================
# STANDARD UPGRADE STEPS
#=================================================
# INSTALL DEPENDENCIES
#=================================================
# Disable swapfile
if [ -z ${tmp_swap_file+x} ];
# Check depends installation
ynh_install_app_dependencies python-dev python-virtualenv \
uwsgi uwsgi-plugin-python \
`# ARM support: ` \
build-essential libssl-dev libffi-dev
#=================================================
# DOWNLOAD, CHECK AND UNPACK SOURCE
#=================================================
# Download, check integrity, uncompress and patch the source from app.src
ynh_setup_source "$final_path"
#=================================================
# NGINX CONFIGURATION
#=================================================
# Create a dedicated nginx config
ynh_add_nginx_config
if [ "$path_url" == "/" ]
then
sudo swapoff $tmp_swap_file
sudo rm -f $tmp_swap_file
# $finalnginxconf comes from ynh_add_nginx_config
# uwsgi_param is only needed for non-root installation
ynh_replace_string "uwsgi_param " "#uwsgi_param " "$finalnginxconf"
ynh_replace_string "uwsgi_modifier1 " "#uwsgi_modifier1 " "$finalnginxconf"
fi
ynh_store_file_checksum "$finalnginxconf"
systemctl reload nginx
#=================================================
# CREATE DEDICATED USER
#=================================================
# Create a system user
ynh_system_user_create "$app"
#=================================================
# pip installation
#=================================================
virtualenv "$final_path/local"
# Init virtualenv
(
set +o nounset
source "$final_path/local/bin/activate"
set -o nounset
cd "$final_path"
pip install --upgrade pip
pip install pyramid_chameleon
CFLAGS="-Wno-error -Wno-error=format-security" \
ARCHFLAGS="-Wno-error=unused-command-line-argument-hard-error-in-future" \
pip install --requirement "$final_path/requirements.txt"
python "$final_path/setup.py" develop
touch "$final_path/local/COMPLETE"
)
# Add nice homepage
cp -r ../sources/page $final_path/syncserver/
(cd "$final_path/syncserver" && patch -p1 < $YNH_CWD/../sources/homepage.patch) || echo "Unable to apply patches"
#=================================================
# SECURE FILES AND DIRECTORIES
#=================================================
chown $app -R $final_path
chmod u=rwX,g=rX,o= -R $final_path
#=================================================
# SETUP UWSGI
#=================================================
# create config file syncserver.ini
rm "$final_path/syncserver.ini"
ln -s "/etc/uwsgi/apps-available/$app.ini" "$final_path/syncserver.ini"
# configure uwsgi
ynh_add_uwsgi_service 'domain secret db_user db_pwd db_name'
#=================================================
# GENERIC FINALIZATION
#=================================================
# SETUP SSOWAT
#=================================================
# Fix permission
sudo find $final_path/ -type d -exec chmod 2755 {} \;
sudo find $final_path/ -type f -exec chmod g+r,o+r {} \;
sudo usermod -a -G ffsync www-data
#enable services
sudo chmod +x /etc/init.d/ffsync
sudo update-rc.d ffsync defaults
sudo service ffsync restart
sudo service ffsync restart
sudo service ffsync restart
# Reload Nginx and regenerate SSOwat conf
sudo service nginx reload
ynh_app_setting_set ffsync skipped_uris "/"
sudo yunohost app ssowatconf
ynh_app_setting_set "$app" skipped_uris "/"

@ -1,48 +0,0 @@
##########################################################
# /!\ WARNING /!\ #
# This is completely experimental. Use at your own risk. #
# Also, learn you some docker: #
# http://docker.io/gettingstarted #
##########################################################
FROM debian:7.4
MAINTAINER Dan Callahan <dan.callahan@gmail.com>
# Base system setup
RUN DEBIAN_FRONTEND=noninteractive apt-get update \
&& apt-get install --no-install-recommends -y \
vim locales \
&& apt-get clean
RUN locale-gen C.UTF-8 && LANG=C.UTF-8 /usr/sbin/update-locale
ENV LANG C.UTF-8
RUN useradd --create-home app
# Build the Sync server
RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
ca-certificates \
build-essential \
libzmq-dev \
python-dev \
python-virtualenv \
&& apt-get clean
USER app
RUN mkdir -p /home/app/syncserver
ADD Makefile *.ini *.wsgi *.rst *.txt *.py /home/app/syncserver/
ADD ./syncserver/ /home/app/syncserver/syncserver/
WORKDIR /home/app/syncserver
RUN make build
# Run the Sync server
EXPOSE 5000
ENTRYPOINT ["/usr/bin/make"]
CMD ["serve"]

@ -1,3 +0,0 @@
include syncserver.ini
include syncserver.wsgi
include syncserver/tests.ini

@ -1,50 +0,0 @@
SYSTEMPYTHON = `which python2 python | head -n 1`
VIRTUALENV = virtualenv --python=$(SYSTEMPYTHON)
ENV = ./local
TOOLS := $(addprefix $(ENV)/bin/,flake8 nosetests)
# Hackety-hack around OSX system python bustage.
# The need for this should go away with a future osx/xcode update.
ARCHFLAGS = -Wno-error=unused-command-line-argument-hard-error-in-future
# Hackety-hack around errors duing compile of ultramemcached.
CFLAGS = "-Wno-error -Wno-error=format-security"
INSTALL = CFLAGS=$(CFLAGS) ARCHFLAGS=$(ARCHFLAGS) $(ENV)/bin/pip install
.PHONY: all
all: build
.PHONY: build
build: | $(ENV)/COMPLETE
$(ENV)/COMPLETE: requirements.txt
$(VIRTUALENV) --no-site-packages $(ENV)
$(INSTALL) -r requirements.txt
$(ENV)/bin/python ./setup.py develop
touch $(ENV)/COMPLETE
.PHONY: test
test: | $(TOOLS)
$(ENV)/bin/flake8 ./syncserver
$(ENV)/bin/nosetests -s syncstorage.tests
# Tokenserver tests currently broken due to incorrect file paths
# $(ENV)/bin/nosetests -s tokenserver.tests
# Test against a running server
$(ENV)/bin/gunicorn --paste syncserver/tests.ini 2> /dev/null & SERVER_PID=$$!; \
sleep 2; \
$(ENV)/bin/python -m syncstorage.tests.functional.test_storage \
--use-token-server http://localhost:5000/token/1.0/sync/1.5; \
kill $$SERVER_PID
$(TOOLS): | $(ENV)/COMPLETE
$(INSTALL) nose flake8
.PHONY: serve
serve: | $(ENV)/COMPLETE
$(ENV)/bin/gunicorn --paste ./syncserver.ini
.PHONY: clean
clean:
rm -rf $(ENV)

@ -1,73 +0,0 @@
Run-Your-Own Firefox Sync Server
================================
This is an all-in-one package for running a self-hosted Firefox Sync server.
If bundles the "tokenserver" project for authentication and the "syncstorage"
project for storage, produce a single stand-alone webapp.
Complete installation instructions are available at:
https://docs.services.mozilla.com/howtos/run-sync-1.5.html
Quickstart
----------
The Sync Server software runs using **python 2.6** or later, and the build
process requires **make** and **virtualenv**. You will need to have the
following packages (or similar, depending on your operating system) installed:
- python2.7
- python2.7-dev
- python-virtualenv
- make
Take a checkout of this repository, then run "make build" to pull in the
necessary python package dependencies::
$ git clone https://github.com/mozilla-services/syncserver
$ cd syncserver
$ make build
To sanity-check that things got installed correctly, do the following::
$ make test
Now you can run the server::
$ make serve
This should start a server on http://localhost:5000/.
Now go into Firefox's `about:config` page, search for a setting named
"tokenServerURI", and change it to point to your server::
services.sync.tokenServerURI: http://localhost:5000/token/1.0/sync/1.5
Firefox should now sync against your local server rather than the default
Mozilla-hosted servers.
For more details on setting up a stable deployment, see:
https://docs.services.mozilla.com/howtos/run-sync-1.5.html
Customization
-------------
All customization of the server can be done by editing the file
"syncserver.ini", which contains lots of comments to help you on
your way. Things you might like to change include:
* The client-visible hostname for your server. Edit the "public_url"
key under the [syncstorage] section.
* The database in which to store sync data. Edit the "sqluri" setting
under the [syncstorage] section.
Questions, Feedback
-------------------
- IRC channel: #sync. See http://irc.mozilla.org/
- Mailing list: https://mail.mozilla.org/listinfo/services-dev

@ -0,0 +1,48 @@
diff -Naur a/__init__.py b/__init__.py
--- a/__init__.py 2018-08-22 17:54:00.369070610 +0200
+++ b/__init__.py 2018-08-22 20:11:39.861463512 +0200
@@ -11,6 +11,10 @@
from pyramid.response import Response
from pyramid.events import NewRequest, subscriber
+from pyramid.static import static_view
+from pyramid.view import view_config
+from pyramid.renderers import render, render_to_response
+
try:
import requests.packages.urllib3.contrib.pyopenssl
HAS_PYOPENSSL = True
@@ -135,13 +139,27 @@
config.scan("syncserver", ignore=["syncserver.wsgi_app"])
config.include("syncstorage", route_prefix="/storage")
config.include("tokenserver", route_prefix="/token")
+ config.include('pyramid_chameleon')
- # Add a top-level "it works!" view.
- def itworks(request):
- return Response("it works!")
-
- config.add_route('itworks', '/')
- config.add_view(itworks, route_name='itworks')
+ # Add a top-level explaination view.
+ # First view, available at http://localhost:6543/
+ def page(request):
+ result = render('page/index.pt',
+ {'public_url':public_url},
+ request=request)
+ response = Response(result)
+ return response
+ config.add_route('page', '/')
+ config.add_view(page, route_name='page')
+
+ www = static_view(
+ os.path.realpath(os.path.dirname(__file__)+"/page/"),
+ use_subpath=True
+ )
+ # Documentation for Hybrid routing can be found here
+ # http://docs.pylonsproject.org/projects/pyramid/en/1.0-branch/narr/hybrid.html#using-subpath-in-a-route-pattern
+ config.add_route('index', '/*subpath', 'www') # subpath is a reserved word
+ config.add_view(www, route_name='index')
def import_settings_from_environment_variables(settings, environ=None):

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 14 KiB

Before

Width:  |  Height:  |  Size: 2.8 KiB

After

Width:  |  Height:  |  Size: 2.8 KiB

@ -1,13 +0,0 @@
cornice==0.16.2
gunicorn==19.1.1
pyramid==1.5
pyramid_chameleon==0.3
requests==2.7
simplejson==3.4
SQLAlchemy==0.9.4
unittest2==0.5.1
zope.component==4.2.1
configparser==3.5.0b2
https://github.com/mozilla-services/mozservices/archive/e00e1b68130423ad98d0f6185655bde650443da8.zip
https://github.com/mozilla-services/tokenserver/archive/d7e513e8a4f5c588b70d685a8df1d2e508c341c0.zip
http://github.com/mozilla-services/server-syncstorage/archive/1.5.5.zip

@ -1,16 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from setuptools import setup
entry_points = """
[paste.app_factory]
main = syncserver:main
"""
setup(
name='syncserver',
version="1.5.2",
packages=['syncserver'],
entry_points=entry_points
)

@ -1,40 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import sys
import site
from logging.config import fileConfig
from ConfigParser import NoSectionError
# detecting if virtualenv was used in this dir
_CURDIR = os.path.dirname(os.path.abspath(__file__))
_PY_VER = sys.version.split()[0][:3]
_SITE_PKG = os.path.join(_CURDIR, 'local', 'lib', 'python' + _PY_VER, 'site-packages')
# adding virtualenv's site-package and ordering paths
saved = sys.path[:]
if os.path.exists(_SITE_PKG):
site.addsitedir(_SITE_PKG)
for path in sys.path:
if path not in saved:
saved.insert(0, path)
sys.path[:] = saved
# setting up the egg cache to a place where apache can write
os.environ['PYTHON_EGG_CACHE'] = '/tmp/python-eggs'
# setting up logging
ini_file = os.path.join(_CURDIR, 'syncserver.ini')
try:
fileConfig(ini_file)
except NoSectionError:
pass
# running the app using Paste
from paste.deploy import loadapp
application = loadapp('config:%s'% ini_file)

@ -1,174 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import logging
from urlparse import urlparse, urlunparse
from pyramid.events import NewRequest, subscriber
from pyramid.static import static_view
from pyramid.view import view_config
from pyramid.renderers import render, render_to_response
from pyramid.response import Response
import mozsvc.config
from tokenserver.util import _JSONError
logger = logging.getLogger("syncserver")
def includeme(config):
"""Install SyncServer application into the given Pyramid configurator."""
# Set the umask so that files are created with secure permissions.
# Necessary for e.g. created-on-demand sqlite database files.
os.umask(0077)
# Sanity-check the deployment settings and provide sensible defaults.
settings = config.registry.settings
public_url = settings.get("syncserver.public_url")
if public_url is None:
raise RuntimeError("you much configure syncserver.public_url")
public_url = public_url.rstrip("/")
settings["syncserver.public_url"] = public_url
secret = settings.get("syncserver.secret")
if secret is None:
secret = os.urandom(32).encode("hex")
sqluri = settings.get("syncserver.sqluri")
if sqluri is None:
rootdir = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
sqluri = "sqlite:///" + os.path.join(rootdir, "syncserver.db")
# Configure app-specific defaults based on top-level configuration.
settings.pop("config", None)
if "tokenserver.backend" not in settings:
# Default to our simple static node-assignment backend
settings["tokenserver.backend"] =\
"syncserver.staticnode.StaticNodeAssignment"
settings["tokenserver.sqluri"] = sqluri
settings["tokenserver.node_url"] = public_url
settings["endpoints.sync-1.5"] = "{node}/storage/1.5/{uid}"
if "tokenserver.monkey_patch_gevent" not in settings:
# Default to no gevent monkey-patching
settings["tokenserver.monkey_patch_gevent"] = False
if "tokenserver.applications" not in settings:
# Default to just the sync-1.5 application
settings["tokenserver.applications"] = "sync-1.5"
if "tokenserver.secrets.backend" not in settings:
# Default to a single fixed signing secret
settings["tokenserver.secrets.backend"] = "mozsvc.secrets.FixedSecrets"
settings["tokenserver.secrets.secrets"] = [secret]
if "tokenserver.allow_new_users" not in settings:
allow_new_users = settings.get("syncserver.allow_new_users")
if allow_new_users is not None:
settings["tokenserver.allow_new_users"] = allow_new_users
if "hawkauth.secrets.backend" not in settings:
# Default to the same secrets backend as the tokenserver
for key in settings.keys():
if key.startswith("tokenserver.secrets."):
newkey = "hawkauth" + key[len("tokenserver"):]
settings[newkey] = settings[key]
if "storage.backend" not in settings:
# Default to sql syncstorage backend
settings["storage.backend"] = "syncstorage.storage.sql.SQLStorage"
settings["storage.sqluri"] = sqluri
settings["storage.create_tables"] = True
if "browserid.backend" not in settings:
# Default to remote verifier, with base of public_url as only audience
audience = urlunparse(urlparse(public_url)._replace(path=""))
settings["browserid.backend"] = "tokenserver.verifiers.RemoteVerifier"
settings["browserid.audiences"] = audience
if "loggers" not in settings:
# Default to basic logging config.
root_logger = logging.getLogger("")
if not root_logger.handlers:
logging.basicConfig(level=logging.INFO)
# Include the relevant sub-packages.
config.scan("syncserver")
config.include("syncstorage", route_prefix="/storage")
config.include("tokenserver", route_prefix="/token")
config.include('pyramid_chameleon')
# Add a top-level explaination view.
# First view, available at http://localhost:6543/
def page(request):
result = render('page/index.pt',
{'public_url':public_url},
request=request)
response = Response(result)
return response
config.add_route('page', '/')
config.add_view(page, route_name='page')
www = static_view(
os.path.realpath(os.path.dirname(__file__)+"/page/"),
use_subpath=True
)
# Documentation for Hybrid routing can be found here
# http://docs.pylonsproject.org/projects/pyramid/en/1.0-branch/narr/hybrid.html#using-subpath-in-a-route-pattern
config.add_route('index', '/*subpath', 'www') # subpath is a reserved word
config.add_view(www, route_name='index')
@subscriber(NewRequest)
def reconcile_wsgi_environ_with_public_url(event):
"""Event-listener that checks and tweaks WSGI environ based on public_url.
This is a simple trick to help ensure that the configured public_url
matches the actual deployed address. It fixes fixes parts of the WSGI
environ where it makes sense (e.g. SCRIPT_NAME) and warns about any parts
that seem obviously mis-configured (e.g. http:// versus https://).
It's very important to get public_url and WSGI environ matching exactly,
since they're used for browserid audience checking and HAWK signature
validation, so mismatches can easily cause strange and cryptic errors.
"""
request = event.request
public_url = request.registry.settings["syncserver.public_url"]
p_public_url = urlparse(public_url)
# If we don't have a SCRIPT_NAME, take it from the public_url.
# This is often the case if we're behind e.g. an nginx proxy that
# is serving us at some sub-path.
if not request.script_name:
request.script_name = p_public_url.path.rstrip("/")
# If the environ does not match public_url, requests are almost certainly
# going to fail due to auth errors. We can either bail out early, or we
# can forcibly clobber the WSGI environ with the values from public_url.
# This is a security risk if you've e.g. mis-configured the server, so
# it's not enabled by default.
application_url = request.application_url
if public_url != application_url:
if not request.registry.settings.get("syncserver.force_wsgi_environ"):
msg = "\n".join((
"The public_url setting doesn't match the application url.",
"This will almost certainly cause authentication failures!",
" public_url setting is: %s" % (public_url,),
" application url is: %s" % (application_url,),
"You can disable this check by setting the force_wsgi_environ",
"option in your config file, but do so at your own risk.",
))
logger.error(msg)
raise _JSONError([msg], status_code=500)
request.scheme = p_public_url.scheme
request.host = p_public_url.netloc
request.script_name = p_public_url.path.rstrip("/")
def get_configurator(global_config, **settings):
"""Load a SyncStorge configurator object from deployment settings."""
config = mozsvc.config.get_configurator(global_config, **settings)
config.begin()
try:
config.include(includeme)
finally:
config.end()
return config
def main(global_config, **settings):
"""Load a SyncStorage WSGI app from deployment settings."""
config = get_configurator(global_config, **settings)
return config.make_wsgi_app()

@ -1,221 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Simple node-assignment backend using a single, static node.
This is a greatly-simplified node-assignment backend. It keeps user records
in an SQL database, but does not attempt to do any node management. All users
are implicitly assigned to a single, static node.
XXX TODO: move this into the tokenserver repo.
"""
import time
import urlparse
from mozsvc.exceptions import BackendError
from sqlalchemy import Column, Integer, String, BigInteger, Index
from sqlalchemy import create_engine, Table, MetaData
from sqlalchemy.pool import QueuePool
from sqlalchemy.sql import text as sqltext
from sqlalchemy.exc import IntegrityError
from tokenserver.assignment import INodeAssignment
from zope.interface import implements
metadata = MetaData()
users = Table(
"users",
metadata,
Column("uid", Integer(), primary_key=True, autoincrement=True,
nullable=False),
Column("service", String(32), nullable=False),
Column("email", String(255), nullable=False),
Column("generation", BigInteger(), nullable=False),
Column("client_state", String(32), nullable=False),
Column("created_at", BigInteger(), nullable=False),
Column("replaced_at", BigInteger(), nullable=True),
Index('lookup_idx', 'email', 'service', 'created_at'),
Index('clientstate_idx', 'email', 'service', 'client_state', unique=True),
)
_GET_USER_RECORDS = sqltext("""\
select
uid, generation, client_state
from
users
where
email = :email
and
service = :service
order by
created_at desc, uid desc
limit
20
""")
_CREATE_USER_RECORD = sqltext("""\
insert into
users
(service, email, generation, client_state, created_at, replaced_at)
values
(:service, :email, :generation, :client_state, :timestamp, NULL)
""")
_UPDATE_GENERATION_NUMBER = sqltext("""\
update
users
set
generation = :generation
where
service = :service and email = :email and
generation < :generation and replaced_at is null
""")
_REPLACE_USER_RECORDS = sqltext("""\
update
users
set
replaced_at = :timestamp
where
service = :service and email = :email
and replaced_at is null and created_at < :timestamp
""")
def get_timestamp():
"""Get current timestamp in milliseconds."""
return int(time.time() * 1000)
class StaticNodeAssignment(object):
implements(INodeAssignment)
def __init__(self, sqluri, node_url, **kw):
self.sqluri = sqluri
self.node_url = node_url
self.driver = urlparse.urlparse(sqluri).scheme.lower()
sqlkw = {
"logging_name": "syncserver",
"connect_args": {},
"poolclass": QueuePool,
"pool_reset_on_return": True,
}
if self.driver == "sqlite":
# We must mark it as safe to share sqlite connections between
# threads. The pool will ensure there's on race conditions.
sqlkw["connect_args"]["check_same_thread"] = False
# If using a :memory: database, we must use a QueuePool of size
# 1 so that a single connection is shared by all threads.
if urlparse.urlparse(sqluri).path.lower() in ("/", "/:memory:"):
sqlkw["pool_size"] = 1
sqlkw["max_overflow"] = 0
if "mysql" in self.driver:
# Guard against the db closing idle conections.
sqlkw["pool_recycle"] = 3600
self._engine = create_engine(sqluri, **sqlkw)
users.create(self._engine, checkfirst=True)
def get_user(self, service, email):
params = {'service': service, 'email': email}
res = self._engine.execute(_GET_USER_RECORDS, **params)
try:
row = res.fetchone()
if row is None:
return None
# The first row is the most up-to-date user record.
user = {
'email': email,
'uid': row.uid,
'node': self.node_url,
'generation': row.generation,
'client_state': row.client_state,
'old_client_states': {}
}
# Any subsequent rows are due to old client-state values.
row = res.fetchone()
while row is not None:
user['old_client_states'][row.client_state] = True
row = res.fetchone()
return user
finally:
res.close()
def allocate_user(self, service, email, generation=0, client_state=''):
params = {
'service': service, 'email': email, 'generation': generation,
'client_state': client_state, 'timestamp': get_timestamp()
}
try:
res = self._engine.execute(_CREATE_USER_RECORD, **params)
except IntegrityError:
raise
return self.get_user(service, email)
else:
res.close()
return {
'email': email,
'uid': res.lastrowid,
'node': self.node_url,
'generation': generation,
'client_state': client_state,
'old_client_states': {}
}
def update_user(self, service, user, generation=None, client_state=None):
if client_state is None:
# uid can stay the same, just update the generation number.
if generation is not None:
params = {
'service': service,
'email': user['email'],
'generation': generation,
}
res = self._engine.execute(_UPDATE_GENERATION_NUMBER, **params)
res.close()
user['generation'] = max(generation, user['generation'])
else:
# reject previously-seen client-state strings.
if client_state == user['client_state']:
raise BackendError('previously seen client-state string')
if client_state in user['old_client_states']:
raise BackendError('previously seen client-state string')
# need to create a new record for new client_state.
if generation is not None:
generation = max(user['generation'], generation)
else:
generation = user['generation']
now = get_timestamp()
params = {
'service': service, 'email': user['email'],
'generation': generation, 'client_state': client_state,
'timestamp': now,
}
try:
res = self._engine.execute(_CREATE_USER_RECORD, **params)
except IntegrityError:
user.update(self.get_user(service, user['email']))
else:
self.get_user(service, user['email'])
user['uid'] = res.lastrowid
user['generation'] = generation
user['old_client_states'][user['client_state']] = True
user['client_state'] = client_state
res.close()
# mark old records as having been replaced.
# if we crash here, they are unmarked and we may fail to
# garbage collect them for a while, but the active state
# will be undamaged.
params = {
'service': service, 'email': user['email'], 'timestamp': now
}
res = self._engine.execute(_REPLACE_USER_RECORDS, **params)
res.close()

@ -1,19 +0,0 @@
[server:main]
use = egg:gunicorn
host = 0.0.0.0
port = 5000
workers = 1
timeout = 30
[app:main]
use = egg:SyncServer
[syncserver]
# This must be edited to point to the public URL of your server.
public_url = http://localhost:5000/
# This defines the database in which to store all server data.
#sqluri = sqlite:////tmp/syncserver.db
# This is a secret key used for signing authentication tokens.
#secret = INSERT_SECRET_KEY_HERE
Loading…
Cancel
Save