diff --git a/README.md b/README.md index 117e70a..c0cb90c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,18 @@ # Mozilla’s Sync Server for Yunohost -The Sync Server provides a replacement for Firefox’s default server (hosted at Mozilla). Its code is available on [the Sync-1.5 Server documentation](https://docs.services.mozilla.com/howtos/run-sync-1.5.html). +The Sync Server provides a replacement for Firefox’s default server (hosted at Mozilla). By default, a server set up will defer authentication to the Mozilla-hosted accounts server at [https://accounts.firefox.com](https://accounts.firefox.com). So you will still have to authenticate at Mozilla, but _the storage of your information will be done on your host_. -**Shipped version:** 1.5 +**Shipped version:** 1.8 ## Configuring Once installed, reaching `http://domain.tld/path` should show a page explaining how to configure it. Otherwise please refer to the [Yunohost page](https://yunohost.org/#/app_ffsync). -## Contributing +## Links -From the `sources` directory, do as follows: - -`make build` - -`make test` - -`./local/bin/gunicorn --reload --paste syncserver.ini` instead of the classical `make serve` to take into account changes you will surely do by developping the app. + * Report a bug about this package: https://github.com/YunoHost-Apps/ffsync_ynh + * Report a bug about Funkwhale itself: https://github.com/mozilla-services/syncserver + * Documentation: https://docs.services.mozilla.com/howtos/run-sync-1.5.html + * YunoHost website: https://yunohost.org/ \ No newline at end of file diff --git a/check_process b/check_process new file mode 100644 index 0000000..1b21f7b --- /dev/null +++ b/check_process @@ -0,0 +1,41 @@ +;; Test name + ; Manifest + domain="domain.tld" (DOMAIN) + path="/path" (PATH) + ; Checks + pkg_linter=1 + setup_sub_dir=1 + setup_root=1 + setup_nourl=0 + setup_private=0 + setup_public=1 + upgrade=1 + upgrade=1 from_commit=fd6350495d5a1d864ae30e1a61e18939fdb6a428 + upgrade=1 from_commit=267ccc21f7b52d22bc3d5b9cd6239857b9a82aad + backup_restore=1 + multi_instance=1 + incorrect_path=1 + port_already_use=0 + change_url=0 +;;; Levels + Level 1=auto + Level 2=auto + Level 3=auto + # impossible with Firefox Sync (it uses Firefox Accounts) + Level 4=na + Level 5=auto + Level 6=auto + Level 7=auto + Level 8=0 + Level 9=0 + Level 10=0 +;;; Options +Email=jean-baptiste@holcroft.fr +Notification=fail +;;; Upgrade options + ; commit=fd6350495d5a1d864ae30e1a61e18939fdb6a428 + name=latest published version + manifest_arg=domain=DOMAIN&path=PATH + ; commit=267ccc21f7b52d22bc3d5b9cd6239857b9a82aad + name=latest git commit before Jibec's rewriting + manifest_arg=domain=DOMAIN&path=PATH diff --git a/conf/app.src b/conf/app.src new file mode 100644 index 0000000..28519e2 --- /dev/null +++ b/conf/app.src @@ -0,0 +1,4 @@ +SOURCE_URL=https://github.com/mozilla-services/syncserver/archive/1.8.0.tar.gz +SOURCE_SUM=728206bcffec7a305e97e7cd4d465b3fa56f39f8e1fd55d98e49a866016d61e0 +SOURCE_SUM_PRG=sha256sum +SOURCE_FORMAT=tar.gz diff --git a/conf/ffsync b/conf/ffsync deleted file mode 100755 index e2fb135..0000000 --- a/conf/ffsync +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -# /etc/init.d/sync -# version 0.1 2013-03-12 (YYYY-MM-DD) - -### BEGIN INIT INFO -# Provides: sync -# Required-Start: $local_fs $remote_fs -# Required-Stop: $local_fs $remote_fs -# Should-Start: $network -# Should-Stop: $network -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Mozilla Sync server -# Description: Starts the mozilla sync server -### END INIT INFO - -# Source function library. -#. /etc/rc.d/init.d/functions - -prog=sync -SYNC_USER=ffsync -SYNC_HOME=/opt/yunohost/ffsync -CPU_COUNT=2 -pidfile=/tmp/sync.pid -lockfile=/var/run/sync.lock -conffile=${SYNC_HOME}/syncserver.ini -GUNICORN=${SYNC_HOME}/local/bin/gunicorn -GUNICORN_ARGS="--paste $conffile --access-logfile /var/log/ffsync.log --daemon -p $pidfile" - -start () { - echo -n "Starting $prog" - start-stop-daemon --start -c ${SYNC_USER} --exec $GUNICORN -- $GUNICORN_ARGS - RETVAL=$? - echo - [ $RETVAL = 0 ] && touch ${lockfile} - return $RETVAL -} - -stop() { - echo "Stopping $prog" - start-stop-daemon --stop --quiet --oknodo --pidfile ${pidfile} - #log_end_msg $? - rm -f ${pidfile} -} - -status(){ - if [[ -f ${pidfile} ]]; then - echo "Status: running." - exit 0; - else - echo "Status: not running." - exit 1; - fi -} - -case "$1" in - start) - start - ;; - stop) - stop - ;; - status) - status - ;; - restart) - stop - start - ;; - *) - echo $"Usage: $prog {start|stop|restart|help}" - RETVAL=2 -esac - -exit $RETVAL diff --git a/conf/ffsync.logrotate b/conf/ffsync.logrotate deleted file mode 100644 index c5f42b7..0000000 --- a/conf/ffsync.logrotate +++ /dev/null @@ -1,9 +0,0 @@ -"/var/log/ffsync.log" { - copytruncate - daily - rotate 7 - compress - delaycompress - missingok - notifempty -} diff --git a/conf/nginx.conf b/conf/nginx.conf index ab6e584..0019584 100644 --- a/conf/nginx.conf +++ b/conf/nginx.conf @@ -1,15 +1,17 @@ -location PATHTOCHANGE { - if ($scheme = http) { - rewrite ^ https://$server_name$request_uri? permanent; - } - proxy_set_header Host $http_host; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Real-IP $remote_addr; - proxy_redirect off; - proxy_read_timeout 120; - proxy_connect_timeout 10; - proxy_pass http://127.0.0.1:5000/; +#sub_path_only rewrite ^__PATH__$ __PATH__/ permanent; - include conf.d/yunohost_panel.conf.inc; +location __PATH__/ { + # Path to source + alias __FINALPATH__/ ; + + if ($scheme = http) { + rewrite ^ https://$server_name$request_uri? permanent; + } + + include uwsgi_params; + # Needed for long running operations in admin interface + uwsgi_read_timeout 3600; + uwsgi_param SCRIPT_NAME __PATH__; + uwsgi_modifier1 30; + uwsgi_pass unix:///var/run/__NAME__/app.socket; } diff --git a/conf/syncserver.ini b/conf/syncserver.ini deleted file mode 100644 index fd5ffa3..0000000 --- a/conf/syncserver.ini +++ /dev/null @@ -1,45 +0,0 @@ -[server:main] -use = egg:gunicorn -host = 0.0.0.0 -port = 5000 -workers = 1 -timeout = 30 - -[app:main] -use = egg:syncserver - -[syncserver] -# This must be edited to point to the public URL of your server, -# i.e. the URL as seen by Firefox. -public_url = https://ynhbaseurl/ - -# This defines the database in which to store all server data. -sqluri = pymysql://yunouser:yunopass@localhost/yunobase - -# This is a secret key used for signing authentication tokens. -# It should be long and randomly-generated. -# The following command will give a suitable value on *nix systems: -# -# head -c 20 /dev/urandom | sha1sum -# -# If not specified then the server will generate a temporary one at startup. -secret = changesecret - -# Set this to "false" to disable new-user signups on the server. -# Only request by existing accounts will be honoured. -allow_new_users = true - -# Set this to "true" to work around a mismatch between public_url and -# the application URL as seen by python, which can happen in certain reverse- -# proxy hosting setups. It will overwrite the WSGI environ dict with the -# details from public_url. This could have security implications if e.g. -# you tell the app that it's on HTTPS but it's really on HTTP, so it should -# only be used as a last resort and after careful checking of server config. -force_wsgi_environ = true - -# Uncomment and edit the following to use a local BrowserID verifier -# rather than posting assertions to the mozilla-hosted verifier. -# Audiences should be set to your public_url without a trailing slash. -#[browserid] -#backend = tokenserver.verifiers.LocalVerifier -#audiences = https://localhost:5000 diff --git a/conf/uwsgi-app@.service b/conf/uwsgi-app@.service new file mode 100644 index 0000000..f61228d --- /dev/null +++ b/conf/uwsgi-app@.service @@ -0,0 +1,20 @@ +[Unit] +Description=%i uWSGI app +After=syslog.target + +[Service] +RuntimeDirectory=%i +ExecStart=/usr/bin/uwsgi \ + --ini /etc/uwsgi/apps-available/%i.ini \ + --socket /var/run/%i/app.socket \ + --logto /var/log/uwsgi/%i/%i.log +User=%i +Group=www-data +Restart=on-failure +KillSignal=SIGQUIT +Type=notify +StandardError=syslog +NotifyAccess=all + +[Install] +WantedBy=multi-user.target diff --git a/sources/syncserver.ini b/conf/uwsgi.ini similarity index 75% rename from sources/syncserver.ini rename to conf/uwsgi.ini index 7672182..9b8c360 100644 --- a/sources/syncserver.ini +++ b/conf/uwsgi.ini @@ -1,9 +1,14 @@ -[server:main] -use = egg:gunicorn -host = 0.0.0.0 -port = 5000 -workers = 1 -timeout = 30 +[uwsgi] +plugins = python +master = true +protocol = uwsgi +socket = /var/run/__APP__/app.socket +chmod-socket = 660 +virtualenv = __FINALPATH__/local +wsgi-file = __FINALPATH__/syncserver.wsgi +python-path = __FINALPATH__/local +enable-threads = true +close-on-exec = true [app:main] use = egg:syncserver @@ -11,10 +16,10 @@ use = egg:syncserver [syncserver] # This must be edited to point to the public URL of your server, # i.e. the URL as seen by Firefox. -public_url = http://localhost:5000/ +public_url = https://__DOMAIN____PATH__ # This defines the database in which to store all server data. -#sqluri = sqlite:////tmp/syncserver.db +sqluri = pymysql://__DB_USER__:__DB_PWD__@localhost/__DB_NAME__ # This is a secret key used for signing authentication tokens. # It should be long and randomly-generated. @@ -23,11 +28,11 @@ public_url = http://localhost:5000/ # head -c 20 /dev/urandom | sha1sum # # If not specified then the server will generate a temporary one at startup. -#secret = INSERT_SECRET_KEY_HERE +secret = __SECRET__ # Set this to "false" to disable new-user signups on the server. # Only request by existing accounts will be honoured. -# allow_new_users = false +allow_new_users = true # Set this to "true" to work around a mismatch between public_url and # the application URL as seen by python, which can happen in certain reverse- @@ -42,4 +47,3 @@ force_wsgi_environ = false # Audiences should be set to your public_url without a trailing slash. #[browserid] #backend = tokenserver.verifiers.LocalVerifier -#audiences = https://localhost:5000 diff --git a/manifest.json b/manifest.json index 32a2bef..6e34ee0 100644 --- a/manifest.json +++ b/manifest.json @@ -1,25 +1,31 @@ { "name": "Firefox Sync Server", "id": "ffsync", - "url": "https://github.com/abeudin/ffsync_ynh", + "version": "1.8.0~ynh1", + "url": "https://github.com/mozilla-services/syncserver", "packaging_format": 1, "description": { "en": "Mozilla’s Sync-Server to host your Firefox account data", "fr": "Le serveur de synchronisation de Mozilla, pour héberger vos données Firefox" }, "maintainer": { + "name": "Josué Tille", + "email": "josue@tille.ch" + }, + "previous_maintainers": [{ "name": "beudbeud", "email": "beudbeud@beudibox.fr", - "url": "https://github.com/balu-/FSyncMS" - }, + "name": "jibec", + "email": "jean-baptiste@holcroft.fr" + }], "requirements": { - "yunohost": ">> 2.4.0" + "yunohost": ">=3.0.0" }, - "license": "free", + "license": "MPL-2.0", "services": [ "nginx" ], - "multi_instance": false, + "multi_instance": true, "arguments": { "install" : [ { diff --git a/scripts/_common.sh b/scripts/_common.sh new file mode 100644 index 0000000..6d7e3bb --- /dev/null +++ b/scripts/_common.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +# Check if system wide templates are available and correcly configured +# +# usage: ynh_check_global_uwsgi_config +ynh_check_global_uwsgi_config () { + uwsgi --version || ynh_die "You need to add uwsgi (and appropriate plugin) as a dependency" + + cp ../conf/uwsgi-app@.service /etc/systemd/system/uwsgi-app@.service + + systemctl daemon-reload +} + +# Create a dedicated uwsgi ini file to use with generic uwsgi service +# +# This will use a template in ../conf/uwsgi.ini +# and will replace the following keywords with +# global variables that should be defined before calling +# this helper : +# +# __APP__ by $app +# __PATH__ by $path_url +# __FINALPATH__ by $final_path +# +# And dynamic variables (from the last example) : +# __PATH_2__ by $path_2 +# __PORT_2__ by $port_2 +# +# usage: ynh_add_uwsgi_service +# +# to interact with your service: `systemctl uwsgi-app@app` +ynh_add_uwsgi_service () { + ynh_check_global_uwsgi_config + + local others_var=${1:-} + local finaluwsgiini="/etc/uwsgi/apps-available/$app.ini" + + # www-data group is needed since it is this nginx who will start the service + usermod --append --groups www-data "$app" || ynh_die "It wasn't possible to add user $app to group www-data" + + ynh_backup_if_checksum_is_different "$finaluwsgiini" + cp ../conf/uwsgi.ini "$finaluwsgiini" + + # To avoid a break by set -u, use a void substitution ${var:-}. If the variable is not set, it's simply set with an empty variable. + # Substitute in a nginx config file only if the variable is not empty + if test -n "${final_path:-}"; then + ynh_replace_string "__FINALPATH__" "$final_path" "$finaluwsgiini" + fi + if test -n "${path_url:-}"; then + ynh_replace_string "__PATH__" "$path_url" "$finaluwsgiini" + fi + if test -n "${app:-}"; then + ynh_replace_string "__APP__" "$app" "$finaluwsgiini" + fi + + # Replace all other variable given as arguments + for var_to_replace in $others_var + do + # ${var_to_replace^^} make the content of the variable on upper-cases + # ${!var_to_replace} get the content of the variable named $var_to_replace + ynh_replace_string "__${var_to_replace^^}__" "${!var_to_replace}" "$finaluwsgiini" + done + + ynh_store_file_checksum "$finaluwsgiini" + + chown $app:root "$finaluwsgiini" + + # make sure the folder for logs exists and set authorizations + mkdir -p /var/log/uwsgi/$app + chown $app:root /var/log/uwsgi/$app + chmod -R u=rwX,g=rX,o= /var/log/uwsgi/$app + + systemctl daemon-reload + systemctl stop "uwsgi-app@$app.service" + systemctl enable "uwsgi-app@$app.service" + systemctl start "uwsgi-app@$app.service" + + # Add as a service + yunohost service add "uwsgi-app@$app" --log "/var/log/uwsgi/$app/$app.log" +} + +# Remove the dedicated uwsgi ini file +# +# usage: ynh_remove_uwsgi_service +ynh_remove_uwsgi_service () { + local finaluwsgiini="/etc/uwsgi/apps-available/$app.ini" + if [ -e "$finaluwsgiini" ]; then + systemctl stop "uwsgi-app@$app.service" + systemctl disable "uwsgi-app@$app.service" + yunohost service remove "uwsgi-app@$app" + + ynh_secure_remove "$finaluwsgiini" + ynh_secure_remove "/var/log/uwsgi/$app" + fi +} diff --git a/scripts/backup b/scripts/backup new file mode 100644 index 0000000..f86bf14 --- /dev/null +++ b/scripts/backup @@ -0,0 +1,62 @@ +#!/bin/bash + +#================================================= +# GENERIC START +#================================================= +# IMPORT GENERIC HELPERS +#================================================= + +source ../settings/scripts/_common.sh +source /usr/share/yunohost/helpers + +#================================================= +# MANAGE SCRIPT FAILURE +#================================================= + +# Exit if an error occurs during the execution of the script +ynh_abort_if_errors + +#================================================= +# LOAD SETTINGS +#================================================= + +app=$YNH_APP_INSTANCE_NAME + +final_path=$(ynh_app_setting_get "$app" final_path) +domain=$(ynh_app_setting_get "$app" domain) +db_name=$(ynh_app_setting_get "$app" db_name) + +#================================================= +# STANDARD BACKUP STEPS +#================================================= +# BACKUP THE APP MAIN DIR +#================================================= + +ynh_backup "$final_path" + +#================================================= +# BACKUP THE NGINX CONFIGURATION +#================================================= + +ynh_backup "/etc/nginx/conf.d/$domain.d/$app.conf" + +#================================================= +# BACKUP THE MYSQL DATABASE +#================================================= + +ynh_mysql_dump_db "$db_name" > db.sql + +#================================================= +# SPECIFIC BACKUP +#================================================= +# Backup Log +#================================================= + +ynh_backup "/var/log/uwsgi/$app" + +#================================================= +# BACKUP THE UWSGI FILES +#================================================= + +ynh_backup "/etc/uwsgi/apps-available/$app.ini" +ynh_backup "/etc/systemd/system/uwsgi-app@.service" diff --git a/scripts/install b/scripts/install index 3132dd4..1764eb3 100644 --- a/scripts/install +++ b/scripts/install @@ -1,105 +1,177 @@ #!/bin/bash -# Source app helpers -. /usr/share/yunohost/helpers +#================================================= +# GENERIC START +#================================================= +# IMPORT GENERIC HELPERS +#================================================= + +source _common.sh +source /usr/share/yunohost/helpers + +#================================================= +# MANAGE SCRIPT FAILURE +#================================================= + +# Exit if an error occurs during the execution of the script +ynh_abort_if_errors + +#================================================= +# RETRIEVE ARGUMENTS FROM THE MANIFEST +#================================================= # Retrieve arguments -domain=$1 -path=$2 +domain=$YNH_APP_ARG_DOMAIN +path_url=$YNH_APP_ARG_PATH -# Check domain/path availability -sudo yunohost app checkurl $domain$path -a ffsync -if [[ ! $? -eq 0 ]]; then - ynh_die -fi +app=$YNH_APP_INSTANCE_NAME +final_path="/opt/yunohost/$app" -# Generate random password -db_pwd=$(head -c 8 /dev/urandom | sha1sum | cut -d " " -f1) +# Normalize the url path syntax +path_url=$(ynh_normalize_url_path "$path_url") -# Use 'FSyncMS' as database name and user -db_user=ffsync +#================================================= +# STORE SETTINGS FROM MANIFEST +#================================================= -# Initialize database and store mysql password for upgrade -sudo yunohost app initdb $db_user -p $db_pwd -ynh_app_setting_set ffsync mysqlpwd $db_pwd +ynh_app_setting_set "$app" final_path "$final_path" -# Generate random password and save -secret=$(head -c 20 /dev/urandom | sha1sum | cut -d " " -f1) -ynh_app_setting_set ffsync secret $secret +#================================================= +# CHECK IF THE APP CAN BE INSTALLED WITH THESE ARGS +#================================================= + +# Check destination directory +test ! -e "$final_path" || ynh_die "This path already contains a folder" + +# Check web path availability +ynh_webpath_available "$domain" "$path_url" +# Register (book) web path +ynh_webpath_register "$app" "$domain" "$path_url" + +#================================================= +# STANDARD MODIFICATIONS +#================================================= + + +#================================================= +# INSTALL DEPENDENCIES +#================================================= # Check depends installation -sudo apt-get install make python-dev python-virtualenv -y +ynh_install_app_dependencies python-dev python-virtualenv \ + uwsgi uwsgi-plugin-python \ + `# ARM support: ` \ + build-essential libssl-dev libffi-dev -# Check Swap -if [ $(sudo swapon -s | wc -l) = 1 ]; -then - # It is NOT possible to setup a swap file on a tmpfs filesystem - mount | grep /tmp | grep tmpfs > /dev/null 2>&1 - if [ $? = 1 ]; - then - tmp_swap_file=/tmp/ffsync_swapfile - else - tmp_swap_file=/var/cache/ffsync_swapfile - fi - sudo dd if=/dev/zero of=$tmp_swap_file bs=1M count=256 - sudo chmod 600 $tmp_swap_file - sudo mkswap $tmp_swap_file - sudo swapon $tmp_swap_file -fi +#================================================= +# CREATE A MYSQL DATABASE +#================================================= + +# Use 'FSyncMS' as database name and user +db_user=$app +db_name=$(ynh_sanitize_dbid $app) +ynh_app_setting_set "$app" db_name "$db_name" +ynh_mysql_setup_db "$db_user" "$db_name" -# Modify assets to take path into account -sudo find ../sources/syncserver/page/sync_files/ -type f -exec sed -i -e "s@media\/img@$path\/media\/img@g" {} \; +#================================================= +# DOWNLOAD, CHECK AND UNPACK SOURCE +#================================================= -# Copy files to the right place -final_path=/opt/yunohost/ffsync -sudo mkdir -p $final_path -sudo cp -a ../sources/* $final_path -sudo cp ../conf/ffsync /etc/init.d/ -sudo cp ../conf/ffsync.logrotate /etc/logrotate.d/ffsync -sudo touch /var/log/ffsync.log +# Download, check integrity, uncompress and patch the source from app.src +ynh_setup_source "$final_path" -# Set permissions to ffsync directory -sudo useradd ffsync -d $final_path -sudo chown ffsync:ffsync -R $final_path -sudo chown ffsync /var/log/ffsync.log +# Modify assets to take path into account +# TODO: try to include this as a patch if still needed +# find ../sources/syncserver/page/sync_files/ -type f -exec sed -i -e "s@media\/img@$path_url\/media\/img@g" {} \; -# Modify Nginx configuration file and copy it to Nginx conf directory -sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf -sed -i "s@ALIASTOCHANGE@$final_path/@g" ../conf/nginx.conf -sudo cp ../conf/nginx.conf /etc/nginx/conf.d/$domain.d/ffsync.conf -sudo cp ../conf/syncserver.ini $final_path/syncserver.ini -sudo sed -i -e "s@ynhbaseurl@$domain$path@g" $final_path/syncserver.ini -sudo sed -i -e "s@changesecret@$secret@g" $final_path/syncserver.ini -sudo sed -i "s/yunouser/$db_user/g" $final_path/syncserver.ini -sudo sed -i "s/yunopass/$db_pwd/g" $final_path/syncserver.ini -sudo sed -i "s/yunobase/$db_user/g" $final_path/syncserver.ini -sudo sed -i -e "s@media\/img@$path\/media\/img@g" $final_path/syncserver/page/sync_files/firefox_sync-bundle.css -sudo sed -i -e "s@media\/img@$path\/media\/img@g" $final_path/syncserver/page/sync_files/responsive-bundle.css +#================================================= +# NGINX CONFIGURATION +#================================================= -# Init virtualenv -cd $final_path && sudo make build && sudo ./local/bin/easy_install gunicorn +# Modify Nginx configuration file and copy it to Nginx conf directory +ynh_add_nginx_config -# Disable swapfile -if [ -z ${tmp_swap_file+x} ]; +if [ "$path_url" == "/" ] then - sudo swapoff $tmp_swap_file - sudo rm -f $tmp_swap_file + # $finalnginxconf comes from ynh_add_nginx_config + # uwsgi_param is only needed for non-root installation + ynh_replace_string "uwsgi_param " "#uwsgi_param " "$finalnginxconf" + ynh_replace_string "uwsgi_modifier1 " "#uwsgi_modifier1 " "$finalnginxconf" fi +ynh_store_file_checksum "$finalnginxconf" +systemctl reload nginx + +#================================================= +# CREATE DEDICATED USER +#================================================= + +ynh_system_user_create "$app" "$final_path" + +#================================================= +# SPECIFIC SETUP +#================================================= +# pip installation +#================================================= + +virtualenv "$final_path/local" +# Init virtualenv +( + set +o nounset + source "$final_path/local/bin/activate" + set -o nounset + cd "$final_path" + pip install --upgrade pip + pip install pyramid_chameleon + CFLAGS="-Wno-error -Wno-error=format-security" \ + ARCHFLAGS="-Wno-error=unused-command-line-argument-hard-error-in-future" \ + pip install --requirement "$final_path/requirements.txt" + + python "$final_path/setup.py" develop + + touch "$final_path/local/COMPLETE" +) + +# Add nice homepage +cp -r ../sources/page $final_path/syncserver/ +(cd "$final_path/syncserver" && patch -p1 < $YNH_CWD/../sources/homepage.patch) || echo "Unable to apply patches" + +#================================================= +# SECURE FILES AND DIRECTORIES +#================================================= + +chown $app -R $final_path +chmod u=rwX,g=rX,o= -R $final_path + + +#================================================= +# SETUP UWSGI +#================================================= + +# Generate random password and save +secret=$(ynh_string_random) +ynh_app_setting_set "$app" secret "$secret" + +# create config file syncserver.ini +rm "$final_path/syncserver.ini" +ln -s "/etc/uwsgi/apps-available/$app.ini" "$final_path/syncserver.ini" + +# configure uwsgi +ynh_add_uwsgi_service 'domain secret db_user db_pwd db_name' + +#================================================= +# MODIFY A CONFIG FILE +#================================================= + +# TODO: fix this css patch +# ynh_replace_string "media\/img@$path_url\/media\/img@g" $final_path/syncserver/page/sync_files/firefox_sync-bundle.css +# ynh_replace_string "media\/img@$path_url\/media\/img@g" $final_path/syncserver/page/sync_files/responsive-bundle.css + +#================================================= +# GENERIC FINALIZATION +#================================================= +# SETUP SSOWAT +#================================================= -# Fix permission -sudo find $final_path/ -type d -exec chmod 2755 {} \; -sudo find $final_path/ -type f -exec chmod g+r,o+r {} \; - -#enable services -sudo chmod +x /etc/init.d/ffsync -sudo update-rc.d ffsync defaults -sudo service ffsync restart -sudo service ffsync restart -sudo service ffsync restart - -# Reload Nginx and regenerate SSOwat conf -sudo yunohost app ssowatconf -sudo service nginx restart -sudo yunohost service add ffsync -l /var/log/ffsync.log -ynh_app_setting_set ffsync skipped_uris "/" +# accessible by everyone (authentification is done by firefox accounts) +ynh_app_setting_set "$app" skipped_uris "/" diff --git a/scripts/remove b/scripts/remove index 984aad9..7624079 100644 --- a/scripts/remove +++ b/scripts/remove @@ -1,22 +1,63 @@ #!/bin/bash -# Source app helpers -. /usr/share/yunohost/helpers - -db_user=ffsync -db_name=ffsync -root_pwd=$(sudo cat /etc/yunohost/mysql) -domain=$(ynh_app_setting_get ffsync domain) - -mysql -u root -p$root_pwd -e "DROP DATABASE $db_name ; DROP USER $db_user@localhost ;" -sudo rm -rf /opt/yunohost/ffsync -sudo rm -f /etc/nginx/conf.d/$domain.d/ffsync.conf -sudo service ffsync stop -sudo update-rc.d ffsync remove -sudo rm /etc/init.d/ffsync -sudo rm /etc/logrotate.d/ffsync -sudo yunohost service remove ffsync - -sudo service nginx reload -sudo userdel ffsync -sudo delgroup ffsync +#================================================= +# GENERIC START +#================================================= +# IMPORT GENERIC HELPERS +#================================================= + +source _common.sh +source /usr/share/yunohost/helpers + +#================================================= +# LOAD SETTINGS +#================================================= + +app=$YNH_APP_INSTANCE_NAME + +domain=$(ynh_app_setting_get "$app" domain) +final_path=$(ynh_app_setting_get "$app" final_path) +db_user=$app +db_name=$app + +#================================================= +# STANDARD REMOVE +#================================================= +# REMOVE UWSGI +#================================================= + +ynh_remove_uwsgi_service + +#================================================= +# REMOVE DEPENDENCIES +#================================================= + +# Remove metapackage and its dependencies +ynh_remove_app_dependencies + +#================================================= +# REMOVE THE MYSQL DATABASE +#================================================= + +ynh_mysql_remove_db "$db_user" "$db_name" + +#================================================= +# REMOVE APP MAIN DIR +#================================================= + +ynh_secure_remove "$final_path" + +#================================================= +# REMOVE NGINX CONFIGURATION +#================================================= + +ynh_remove_nginx_config + +#================================================= +# GENERIC FINALIZATION +#================================================= +# REMOVE DEDICATED USER +#================================================= + +# Delete a system user +ynh_system_user_delete "$app" diff --git a/scripts/restore b/scripts/restore new file mode 100644 index 0000000..541ceeb --- /dev/null +++ b/scripts/restore @@ -0,0 +1,105 @@ +#!/bin/bash + +#================================================= +# GENERIC START +#================================================= +# IMPORT GENERIC HELPERS +#================================================= + +source ../settings/scripts/_common.sh +source /usr/share/yunohost/helpers + +#================================================= +# MANAGE SCRIPT FAILURE +#================================================= + +# Exit if an error occurs during the execution of the script +ynh_abort_if_errors + +#================================================= +# LOAD SETTINGS +#================================================= + +app=$YNH_APP_INSTANCE_NAME + +domain=$(ynh_app_setting_get "$app" domain) +path_url=$(ynh_app_setting_get "$app" path) +final_path=$(ynh_app_setting_get "$app" final_path) +db_name=$(ynh_app_setting_get "$app" db_name) + +#================================================= +# CHECK IF THE APP CAN BE RESTORED +#================================================= + +ynh_webpath_available "$domain" "$path_url" \ + || ynh_die "Path not available: ${domain}${path_url}" +test ! -d "$final_path" \ + || ynh_die "There is already a directory: $final_path " + +#================================================= +# STANDARD RESTORATION STEPS +#================================================= + +# Restore all config and data +ynh_restore + +#================================================= +# RESTORE THE MYSQL DATABASE +#================================================= + +db_pwd=$(ynh_app_setting_get "$app" mysqlpwd) +ynh_mysql_setup_db "$db_name" "$db_name" "$db_pwd" +ynh_mysql_connect_as "$db_name" "$db_pwd" "$db_name" < ./db.sql + +#================================================= +# RECREATE THE DEDICATED USER +#================================================= + +# Create the dedicated user (if not existing) +ynh_system_user_create "$app" + +#================================================= +# RESTORE USER RIGHTS +#================================================= + +chown $app -R $final_path +chmod u=rwX,g=rX,o= -R $final_path + +#================================================= +# SPECIFIC RESTORATION +#================================================= +# REINSTALL DEPENDENCIES +#================================================= + +ynh_install_app_dependencies python-dev python-virtualenv \ + uwsgi uwsgi-plugin-python \ + `# ARM support: ` \ + build-essential libssl-dev libffi-dev + +# set authorizations +chown $app:root /var/log/uwsgi/$app +chmod -R u=rwX,g=rX,o= /var/log/uwsgi/$app + +#================================================= +# RESTORE SERVICE +#================================================= + +usermod --append --groups www-data "$app" + +systemctl daemon-reload +systemctl enable "uwsgi-app@$app.service" + +#================================================= +# ADVERTISE SERVICE IN ADMIN PANEL +#================================================= + +yunohost service add "uwsgi-app@$app.service" --log "/var/log/uwsgi/app/$app" + +#================================================= +# GENERIC FINALIZATION +#================================================= +# RELOAD NGINX AND PHP-FPM +#================================================= + +systemctl start "uwsgi-app@$app.service" +systemctl reload nginx diff --git a/scripts/upgrade b/scripts/upgrade index 3a39026..84eb1d6 100644 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -1,91 +1,182 @@ #!/bin/bash -# Source app helpers -. /usr/share/yunohost/helpers +#================================================= +# GENERIC START +#================================================= +# IMPORT GENERIC HELPERS +#================================================= + +source _common.sh +source /usr/share/yunohost/helpers + +#================================================= +# MANAGE SCRIPT FAILURE +#================================================= + +# Exit if an error occurs during the execution of the script +ynh_abort_if_errors + +#================================================= +# RETRIEVE ARGUMENTS FROM THE MANIFEST +#================================================= + +app=$YNH_APP_INSTANCE_NAME # Retrieve arguments -domain=$(ynh_app_setting_get ffsync domain) -path=$(ynh_app_setting_get ffsync path) -db_pwd=$(ynh_app_setting_get ffsync mysqlpwd) -db_user=ffsync -final_path=/opt/yunohost/ffsync - -# Get secret variable -secret=$(ynh_app_setting_get ffsync secret) -# Get from conf file if not defined -if [[ -z $secret ]] -then - secret=$(sudo grep "secret =" $final_path/syncserver.ini | cut -d" " -f3) - ynh_app_setting_set ffsync secret $secret +domain=$(ynh_app_setting_get "$app" domain) +path_url=$(ynh_app_setting_get "$app" path) +db_name=$(ynh_app_setting_get "$app" db_name) +db_pwd=$(ynh_app_setting_get "$app" mysqlpwd) +db_user=$app +final_path=$(ynh_app_setting_get "$app" final_path) +secret=$(ynh_app_setting_get "$app" secret) + +#================================================= +# ENSURE DOWNWARD COMPATIBILITY +#================================================= + +# If db_name doesn't exist, create it +if [ -z "$db_name" ]; then + db_name=$(ynh_sanitize_dbid "$app") + ynh_app_setting_set $app db_name "$db_name" fi -# Check Swap -if [ $(sudo swapon -s | wc -l) = 1 ]; -then - # It is NOT possible to setup a swap file on a tmpfs filesystem - mount | grep /tmp | grep tmpfs > /dev/null 2>&1 - if [ $? = 1 ]; - then - tmp_swap_file=/tmp/ffsync_swapfile - else - tmp_swap_file=/var/cache/ffsync_swapfile - fi - sudo dd if=/dev/zero of=$tmp_swap_file bs=1M count=256 - sudo chmod 600 $tmp_swap_file - sudo mkswap $tmp_swap_file - sudo swapon $tmp_swap_file +# If final_path doesn't exist, create it +if [ -z "$final_path" ]; then + final_path=/opt/yunohost/$app + ynh_app_setting_set "$app" final_path "$final_path" fi -# Copy files to the right place -sudo mkdir -p $final_path -sudo cp -a ../sources/* $final_path -sudo cp ../conf/ffsync /etc/init.d/ -sudo cp ../conf/ffsync.logrotate /etc/logrotate.d/ffsync - - -# Set permissions to ffsync directory -sudo useradd ffsync -d $final_path -sudo chown ffsync:ffsync -R $final_path - -# Modify Nginx configuration file and copy it to Nginx conf directory -sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf -sed -i "s@ALIASTOCHANGE@$final_path@g" ../conf/nginx.conf -sudo cp ../conf/nginx.conf /etc/nginx/conf.d/$domain.d/ffsync.conf -sudo cp ../conf/syncserver.ini $final_path/syncserver.ini -sudo sed -i -e "s@ynhbaseurl@$domain$path@g" $final_path/syncserver.ini -sudo sed -i -e "s@changesecret@$secret@g" $final_path/syncserver.ini -sudo sed -i "s/yunouser/$db_user/g" $final_path/syncserver.ini -sudo sed -i "s/yunopass/$db_pwd/g" $final_path/syncserver.ini -sudo sed -i "s/yunobase/$db_user/g" $final_path/syncserver.ini -sudo sed -i -e "s@media\/img@$path\/media\/img@g" $final_path/syncserver/page/sync_files/firefox_sync-bundle.css -sudo sed -i -e "s@media\/img@$path\/media\/img@g" $final_path/syncserver/page/sync_files/responsive-bundle.css - -# stop service before upgrade -sudo service ffsync stop +# If path_url doesn't exist, create it +if [ -z "$path_url" ]; then + path_url=$(ynh_app_setting_get "$app" path) + ynh_app_setting_set "$app" path "$path_url" + ynh_app_setting_delete "$app" path +fi -# Init virtualenv -cd $final_path && sudo make build && sudo ./local/bin/easy_install gunicorn +# Detect old installation style +if [ -e /etc/init.d/ffsync ]; then + service ffsync stop + update-rc.d -f ffsync remove + + ynh_secure_remove /etc/init.d/ffsync + ynh_secure_remove /var/log/ffsync.log + ynh_secure_remove /opt/yunohost/ffsync + + yunohost service remove "$app" +fi + +#================================================= +# BACKUP BEFORE UPGRADE THEN ACTIVE TRAP +#================================================= + +# Backup the current version of the app +ynh_backup_before_upgrade +ynh_clean_setup () { + # restore it if the upgrade fails + ynh_restore_upgradebackup +} +# Exit if an error occurs during the execution of the script +ynh_abort_if_errors + +#================================================= +# CHECK THE PATH +#================================================= + +# Normalize the URL path syntax +path_url=$(ynh_normalize_url_path "$path_url") + +#================================================= +# STANDARD UPGRADE STEPS +#================================================= +# INSTALL DEPENDENCIES +#================================================= -# Disable swapfile -if [ -z ${tmp_swap_file+x} ]; +# Check depends installation +ynh_install_app_dependencies python-dev python-virtualenv \ + uwsgi uwsgi-plugin-python \ + `# ARM support: ` \ + build-essential libssl-dev libffi-dev + +#================================================= +# DOWNLOAD, CHECK AND UNPACK SOURCE +#================================================= + +# Download, check integrity, uncompress and patch the source from app.src +ynh_setup_source "$final_path" + +#================================================= +# NGINX CONFIGURATION +#================================================= + +# Create a dedicated nginx config +ynh_add_nginx_config + +if [ "$path_url" == "/" ] then - sudo swapoff $tmp_swap_file - sudo rm -f $tmp_swap_file + # $finalnginxconf comes from ynh_add_nginx_config + # uwsgi_param is only needed for non-root installation + ynh_replace_string "uwsgi_param " "#uwsgi_param " "$finalnginxconf" + ynh_replace_string "uwsgi_modifier1 " "#uwsgi_modifier1 " "$finalnginxconf" fi +ynh_store_file_checksum "$finalnginxconf" +systemctl reload nginx + +#================================================= +# CREATE DEDICATED USER +#================================================= + +# Create a system user +ynh_system_user_create "$app" + +#================================================= +# pip installation +#================================================= + +virtualenv "$final_path/local" +# Init virtualenv +( + set +o nounset + source "$final_path/local/bin/activate" + set -o nounset + cd "$final_path" + pip install --upgrade pip + pip install pyramid_chameleon + CFLAGS="-Wno-error -Wno-error=format-security" \ + ARCHFLAGS="-Wno-error=unused-command-line-argument-hard-error-in-future" \ + pip install --requirement "$final_path/requirements.txt" + + python "$final_path/setup.py" develop + + touch "$final_path/local/COMPLETE" +) + +# Add nice homepage +cp -r ../sources/page $final_path/syncserver/ +(cd "$final_path/syncserver" && patch -p1 < $YNH_CWD/../sources/homepage.patch) || echo "Unable to apply patches" + +#================================================= +# SECURE FILES AND DIRECTORIES +#================================================= + +chown $app -R $final_path +chmod u=rwX,g=rX,o= -R $final_path + +#================================================= +# SETUP UWSGI +#================================================= + +# create config file syncserver.ini +rm "$final_path/syncserver.ini" +ln -s "/etc/uwsgi/apps-available/$app.ini" "$final_path/syncserver.ini" + +# configure uwsgi +ynh_add_uwsgi_service 'domain secret db_user db_pwd db_name' + +#================================================= +# GENERIC FINALIZATION +#================================================= +# SETUP SSOWAT +#================================================= -# Fix permission -sudo find $final_path/ -type d -exec chmod 2755 {} \; -sudo find $final_path/ -type f -exec chmod g+r,o+r {} \; -sudo usermod -a -G ffsync www-data - -#enable services -sudo chmod +x /etc/init.d/ffsync -sudo update-rc.d ffsync defaults -sudo service ffsync restart -sudo service ffsync restart -sudo service ffsync restart - -# Reload Nginx and regenerate SSOwat conf -sudo service nginx reload -ynh_app_setting_set ffsync skipped_uris "/" -sudo yunohost app ssowatconf +ynh_app_setting_set "$app" skipped_uris "/" diff --git a/sources/Dockerfile b/sources/Dockerfile deleted file mode 100644 index b66c21f..0000000 --- a/sources/Dockerfile +++ /dev/null @@ -1,48 +0,0 @@ -########################################################## -# /!\ WARNING /!\ # -# This is completely experimental. Use at your own risk. # -# Also, learn you some docker: # -# http://docker.io/gettingstarted # -########################################################## - -FROM debian:7.4 -MAINTAINER Dan Callahan - -# Base system setup - -RUN DEBIAN_FRONTEND=noninteractive apt-get update \ - && apt-get install --no-install-recommends -y \ - vim locales \ - && apt-get clean - -RUN locale-gen C.UTF-8 && LANG=C.UTF-8 /usr/sbin/update-locale - -ENV LANG C.UTF-8 - -RUN useradd --create-home app - -# Build the Sync server - -RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ - ca-certificates \ - build-essential \ - libzmq-dev \ - python-dev \ - python-virtualenv \ - && apt-get clean - -USER app - -RUN mkdir -p /home/app/syncserver -ADD Makefile *.ini *.wsgi *.rst *.txt *.py /home/app/syncserver/ -ADD ./syncserver/ /home/app/syncserver/syncserver/ -WORKDIR /home/app/syncserver - -RUN make build - -# Run the Sync server - -EXPOSE 5000 - -ENTRYPOINT ["/usr/bin/make"] -CMD ["serve"] diff --git a/sources/MANIFEST.in b/sources/MANIFEST.in deleted file mode 100644 index 3ffd5cf..0000000 --- a/sources/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include syncserver.ini -include syncserver.wsgi -include syncserver/tests.ini diff --git a/sources/Makefile b/sources/Makefile deleted file mode 100644 index 2efa163..0000000 --- a/sources/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -SYSTEMPYTHON = `which python2 python | head -n 1` -VIRTUALENV = virtualenv --python=$(SYSTEMPYTHON) -ENV = ./local -TOOLS := $(addprefix $(ENV)/bin/,flake8 nosetests) - -# Hackety-hack around OSX system python bustage. -# The need for this should go away with a future osx/xcode update. -ARCHFLAGS = -Wno-error=unused-command-line-argument-hard-error-in-future - -# Hackety-hack around errors duing compile of ultramemcached. -CFLAGS = "-Wno-error -Wno-error=format-security" - -INSTALL = CFLAGS=$(CFLAGS) ARCHFLAGS=$(ARCHFLAGS) $(ENV)/bin/pip install - - -.PHONY: all -all: build - -.PHONY: build -build: | $(ENV)/COMPLETE -$(ENV)/COMPLETE: requirements.txt - $(VIRTUALENV) --no-site-packages $(ENV) - $(INSTALL) -r requirements.txt - $(ENV)/bin/python ./setup.py develop - touch $(ENV)/COMPLETE - -.PHONY: test -test: | $(TOOLS) - $(ENV)/bin/flake8 ./syncserver - $(ENV)/bin/nosetests -s syncstorage.tests - # Tokenserver tests currently broken due to incorrect file paths - # $(ENV)/bin/nosetests -s tokenserver.tests - - # Test against a running server - $(ENV)/bin/gunicorn --paste syncserver/tests.ini 2> /dev/null & SERVER_PID=$$!; \ - sleep 2; \ - $(ENV)/bin/python -m syncstorage.tests.functional.test_storage \ - --use-token-server http://localhost:5000/token/1.0/sync/1.5; \ - kill $$SERVER_PID - -$(TOOLS): | $(ENV)/COMPLETE - $(INSTALL) nose flake8 - -.PHONY: serve -serve: | $(ENV)/COMPLETE - $(ENV)/bin/gunicorn --paste ./syncserver.ini - -.PHONY: clean -clean: - rm -rf $(ENV) diff --git a/sources/README.rst b/sources/README.rst deleted file mode 100644 index 3872786..0000000 --- a/sources/README.rst +++ /dev/null @@ -1,73 +0,0 @@ -Run-Your-Own Firefox Sync Server -================================ - -This is an all-in-one package for running a self-hosted Firefox Sync server. -If bundles the "tokenserver" project for authentication and the "syncstorage" -project for storage, produce a single stand-alone webapp. - -Complete installation instructions are available at: - - https://docs.services.mozilla.com/howtos/run-sync-1.5.html - - -Quickstart ----------- - -The Sync Server software runs using **python 2.6** or later, and the build -process requires **make** and **virtualenv**. You will need to have the -following packages (or similar, depending on your operating system) installed: - -- python2.7 -- python2.7-dev -- python-virtualenv -- make - -Take a checkout of this repository, then run "make build" to pull in the -necessary python package dependencies:: - - $ git clone https://github.com/mozilla-services/syncserver - $ cd syncserver - $ make build - -To sanity-check that things got installed correctly, do the following:: - - $ make test - -Now you can run the server:: - - $ make serve - -This should start a server on http://localhost:5000/. - -Now go into Firefox's `about:config` page, search for a setting named -"tokenServerURI", and change it to point to your server:: - - services.sync.tokenServerURI: http://localhost:5000/token/1.0/sync/1.5 - -Firefox should now sync against your local server rather than the default -Mozilla-hosted servers. - -For more details on setting up a stable deployment, see: - - https://docs.services.mozilla.com/howtos/run-sync-1.5.html - - -Customization -------------- - -All customization of the server can be done by editing the file -"syncserver.ini", which contains lots of comments to help you on -your way. Things you might like to change include: - - * The client-visible hostname for your server. Edit the "public_url" - key under the [syncstorage] section. - - * The database in which to store sync data. Edit the "sqluri" setting - under the [syncstorage] section. - - -Questions, Feedback -------------------- - -- IRC channel: #sync. See http://irc.mozilla.org/ -- Mailing list: https://mail.mozilla.org/listinfo/services-dev diff --git a/sources/homepage.patch b/sources/homepage.patch new file mode 100644 index 0000000..ecc28dd --- /dev/null +++ b/sources/homepage.patch @@ -0,0 +1,48 @@ +diff -Naur a/__init__.py b/__init__.py +--- a/__init__.py 2018-08-22 17:54:00.369070610 +0200 ++++ b/__init__.py 2018-08-22 20:11:39.861463512 +0200 +@@ -11,6 +11,10 @@ + from pyramid.response import Response + from pyramid.events import NewRequest, subscriber + ++from pyramid.static import static_view ++from pyramid.view import view_config ++from pyramid.renderers import render, render_to_response ++ + try: + import requests.packages.urllib3.contrib.pyopenssl + HAS_PYOPENSSL = True +@@ -135,13 +139,27 @@ + config.scan("syncserver", ignore=["syncserver.wsgi_app"]) + config.include("syncstorage", route_prefix="/storage") + config.include("tokenserver", route_prefix="/token") ++ config.include('pyramid_chameleon') + +- # Add a top-level "it works!" view. +- def itworks(request): +- return Response("it works!") +- +- config.add_route('itworks', '/') +- config.add_view(itworks, route_name='itworks') ++ # Add a top-level explaination view. ++ # First view, available at http://localhost:6543/ ++ def page(request): ++ result = render('page/index.pt', ++ {'public_url':public_url}, ++ request=request) ++ response = Response(result) ++ return response ++ config.add_route('page', '/') ++ config.add_view(page, route_name='page') ++ ++ www = static_view( ++ os.path.realpath(os.path.dirname(__file__)+"/page/"), ++ use_subpath=True ++ ) ++ # Documentation for Hybrid routing can be found here ++ # http://docs.pylonsproject.org/projects/pyramid/en/1.0-branch/narr/hybrid.html#using-subpath-in-a-route-pattern ++ config.add_route('index', '/*subpath', 'www') # subpath is a reserved word ++ config.add_view(www, route_name='index') + + + def import_settings_from_environment_variables(settings, environ=None): diff --git a/sources/syncserver/page/index.pt b/sources/page/index.pt similarity index 100% rename from sources/syncserver/page/index.pt rename to sources/page/index.pt diff --git a/sources/syncserver/page/media/fonts/opensans-bold.5cf854f3d1c0.woff2 b/sources/page/media/fonts/opensans-bold.5cf854f3d1c0.woff2 similarity index 100% rename from sources/syncserver/page/media/fonts/opensans-bold.5cf854f3d1c0.woff2 rename to sources/page/media/fonts/opensans-bold.5cf854f3d1c0.woff2 diff --git a/sources/syncserver/page/media/fonts/opensans-italic.c86748d08341.woff2 b/sources/page/media/fonts/opensans-italic.c86748d08341.woff2 similarity index 100% rename from sources/syncserver/page/media/fonts/opensans-italic.c86748d08341.woff2 rename to sources/page/media/fonts/opensans-italic.c86748d08341.woff2 diff --git a/sources/syncserver/page/media/fonts/opensans-light.2120033991a4.woff2 b/sources/page/media/fonts/opensans-light.2120033991a4.woff2 similarity index 100% rename from sources/syncserver/page/media/fonts/opensans-light.2120033991a4.woff2 rename to sources/page/media/fonts/opensans-light.2120033991a4.woff2 diff --git a/sources/syncserver/page/media/fonts/opensans-lightitalic.580ce2d5ac1d.woff2 b/sources/page/media/fonts/opensans-lightitalic.580ce2d5ac1d.woff2 similarity index 100% rename from sources/syncserver/page/media/fonts/opensans-lightitalic.580ce2d5ac1d.woff2 rename to sources/page/media/fonts/opensans-lightitalic.580ce2d5ac1d.woff2 diff --git a/sources/syncserver/page/media/fonts/opensans-regular.668362de763a.woff2 b/sources/page/media/fonts/opensans-regular.668362de763a.woff2 similarity index 100% rename from sources/syncserver/page/media/fonts/opensans-regular.668362de763a.woff2 rename to sources/page/media/fonts/opensans-regular.668362de763a.woff2 diff --git a/sources/syncserver/page/media/img/firefox/family/nav-sprite.56fbf5a8d218.png b/sources/page/media/img/firefox/family/nav-sprite.56fbf5a8d218.png similarity index 100% rename from sources/syncserver/page/media/img/firefox/family/nav-sprite.56fbf5a8d218.png rename to sources/page/media/img/firefox/family/nav-sprite.56fbf5a8d218.png diff --git a/sources/syncserver/page/media/img/firefox/sync/device-lineup.a60618c7dacb.png b/sources/page/media/img/firefox/sync/device-lineup.a60618c7dacb.png similarity index 100% rename from sources/syncserver/page/media/img/firefox/sync/device-lineup.a60618c7dacb.png rename to sources/page/media/img/firefox/sync/device-lineup.a60618c7dacb.png diff --git a/sources/syncserver/page/media/img/firefox/sync/icons.b80c6430793b.png b/sources/page/media/img/firefox/sync/icons.b80c6430793b.png similarity index 100% rename from sources/syncserver/page/media/img/firefox/sync/icons.b80c6430793b.png rename to sources/page/media/img/firefox/sync/icons.b80c6430793b.png diff --git a/sources/syncserver/page/media/img/sandstone/bg-gradient-sky.7ea325995978.png b/sources/page/media/img/sandstone/bg-gradient-sky.7ea325995978.png similarity index 100% rename from sources/syncserver/page/media/img/sandstone/bg-gradient-sky.7ea325995978.png rename to sources/page/media/img/sandstone/bg-gradient-sky.7ea325995978.png diff --git a/sources/syncserver/page/media/img/sandstone/footer-mozilla.fafef0912042.png b/sources/page/media/img/sandstone/footer-mozilla.fafef0912042.png similarity index 100% rename from sources/syncserver/page/media/img/sandstone/footer-mozilla.fafef0912042.png rename to sources/page/media/img/sandstone/footer-mozilla.fafef0912042.png diff --git a/sources/syncserver/page/media/img/sandstone/grain.855f29e0c686.png b/sources/page/media/img/sandstone/grain.855f29e0c686.png similarity index 100% rename from sources/syncserver/page/media/img/sandstone/grain.855f29e0c686.png rename to sources/page/media/img/sandstone/grain.855f29e0c686.png diff --git a/sources/syncserver/page/media/img/tabzilla/tabzilla-static.953a65a1f4a4.png b/sources/page/media/img/tabzilla/tabzilla-static.953a65a1f4a4.png similarity index 100% rename from sources/syncserver/page/media/img/tabzilla/tabzilla-static.953a65a1f4a4.png rename to sources/page/media/img/tabzilla/tabzilla-static.953a65a1f4a4.png diff --git a/sources/syncserver/page/sync_files/btn-app-store.svg b/sources/page/sync_files/btn-app-store.svg similarity index 100% rename from sources/syncserver/page/sync_files/btn-app-store.svg rename to sources/page/sync_files/btn-app-store.svg diff --git a/sources/syncserver/page/sync_files/btn-google-play.png b/sources/page/sync_files/btn-google-play.png similarity index 100% rename from sources/syncserver/page/sync_files/btn-google-play.png rename to sources/page/sync_files/btn-google-play.png diff --git a/sources/syncserver/page/sync_files/common-bundle.js b/sources/page/sync_files/common-bundle.js similarity index 100% rename from sources/syncserver/page/sync_files/common-bundle.js rename to sources/page/sync_files/common-bundle.js diff --git a/sources/syncserver/page/sync_files/firefox_sync-bundle.css b/sources/page/sync_files/firefox_sync-bundle.css similarity index 100% rename from sources/syncserver/page/sync_files/firefox_sync-bundle.css rename to sources/page/sync_files/firefox_sync-bundle.css diff --git a/sources/syncserver/page/sync_files/firefox_sync-bundle.js b/sources/page/sync_files/firefox_sync-bundle.js similarity index 100% rename from sources/syncserver/page/sync_files/firefox_sync-bundle.js rename to sources/page/sync_files/firefox_sync-bundle.js diff --git a/sources/syncserver/page/sync_files/firefox_sync_anim-bundle.css b/sources/page/sync_files/firefox_sync_anim-bundle.css similarity index 100% rename from sources/syncserver/page/sync_files/firefox_sync_anim-bundle.css rename to sources/page/sync_files/firefox_sync_anim-bundle.css diff --git a/sources/syncserver/page/sync_files/gtm-snippet-bundle.js b/sources/page/sync_files/gtm-snippet-bundle.js similarity index 100% rename from sources/syncserver/page/sync_files/gtm-snippet-bundle.js rename to sources/page/sync_files/gtm-snippet-bundle.js diff --git a/sources/syncserver/page/sync_files/responsive-bundle.css b/sources/page/sync_files/responsive-bundle.css similarity index 100% rename from sources/syncserver/page/sync_files/responsive-bundle.css rename to sources/page/sync_files/responsive-bundle.css diff --git a/sources/syncserver/page/sync_files/site-bundle.js b/sources/page/sync_files/site-bundle.js similarity index 100% rename from sources/syncserver/page/sync_files/site-bundle.js rename to sources/page/sync_files/site-bundle.js diff --git a/sources/requirements.txt b/sources/requirements.txt deleted file mode 100644 index 3223893..0000000 --- a/sources/requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ -cornice==0.16.2 -gunicorn==19.1.1 -pyramid==1.5 -pyramid_chameleon==0.3 -requests==2.7 -simplejson==3.4 -SQLAlchemy==0.9.4 -unittest2==0.5.1 -zope.component==4.2.1 -configparser==3.5.0b2 -https://github.com/mozilla-services/mozservices/archive/e00e1b68130423ad98d0f6185655bde650443da8.zip -https://github.com/mozilla-services/tokenserver/archive/d7e513e8a4f5c588b70d685a8df1d2e508c341c0.zip -http://github.com/mozilla-services/server-syncstorage/archive/1.5.5.zip diff --git a/sources/setup.py b/sources/setup.py deleted file mode 100644 index 8340912..0000000 --- a/sources/setup.py +++ /dev/null @@ -1,16 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. -from setuptools import setup - -entry_points = """ -[paste.app_factory] -main = syncserver:main -""" - -setup( - name='syncserver', - version="1.5.2", - packages=['syncserver'], - entry_points=entry_points -) diff --git a/sources/syncserver.wsgi b/sources/syncserver.wsgi deleted file mode 100644 index 4b0503c..0000000 --- a/sources/syncserver.wsgi +++ /dev/null @@ -1,40 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -import os -import sys -import site -from logging.config import fileConfig -from ConfigParser import NoSectionError - -# detecting if virtualenv was used in this dir -_CURDIR = os.path.dirname(os.path.abspath(__file__)) -_PY_VER = sys.version.split()[0][:3] -_SITE_PKG = os.path.join(_CURDIR, 'local', 'lib', 'python' + _PY_VER, 'site-packages') - -# adding virtualenv's site-package and ordering paths -saved = sys.path[:] - -if os.path.exists(_SITE_PKG): - site.addsitedir(_SITE_PKG) - -for path in sys.path: - if path not in saved: - saved.insert(0, path) - -sys.path[:] = saved - -# setting up the egg cache to a place where apache can write -os.environ['PYTHON_EGG_CACHE'] = '/tmp/python-eggs' - -# setting up logging -ini_file = os.path.join(_CURDIR, 'syncserver.ini') -try: - fileConfig(ini_file) -except NoSectionError: - pass - -# running the app using Paste -from paste.deploy import loadapp -application = loadapp('config:%s'% ini_file) diff --git a/sources/syncserver/__init__.py b/sources/syncserver/__init__.py deleted file mode 100644 index 33731f1..0000000 --- a/sources/syncserver/__init__.py +++ /dev/null @@ -1,174 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -import os -import logging -from urlparse import urlparse, urlunparse - -from pyramid.events import NewRequest, subscriber -from pyramid.static import static_view -from pyramid.view import view_config -from pyramid.renderers import render, render_to_response -from pyramid.response import Response - -import mozsvc.config - -from tokenserver.util import _JSONError - -logger = logging.getLogger("syncserver") - - -def includeme(config): - """Install SyncServer application into the given Pyramid configurator.""" - # Set the umask so that files are created with secure permissions. - # Necessary for e.g. created-on-demand sqlite database files. - os.umask(0077) - - # Sanity-check the deployment settings and provide sensible defaults. - settings = config.registry.settings - public_url = settings.get("syncserver.public_url") - if public_url is None: - raise RuntimeError("you much configure syncserver.public_url") - public_url = public_url.rstrip("/") - settings["syncserver.public_url"] = public_url - - secret = settings.get("syncserver.secret") - if secret is None: - secret = os.urandom(32).encode("hex") - sqluri = settings.get("syncserver.sqluri") - if sqluri is None: - rootdir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) - sqluri = "sqlite:///" + os.path.join(rootdir, "syncserver.db") - - # Configure app-specific defaults based on top-level configuration. - settings.pop("config", None) - if "tokenserver.backend" not in settings: - # Default to our simple static node-assignment backend - settings["tokenserver.backend"] =\ - "syncserver.staticnode.StaticNodeAssignment" - settings["tokenserver.sqluri"] = sqluri - settings["tokenserver.node_url"] = public_url - settings["endpoints.sync-1.5"] = "{node}/storage/1.5/{uid}" - if "tokenserver.monkey_patch_gevent" not in settings: - # Default to no gevent monkey-patching - settings["tokenserver.monkey_patch_gevent"] = False - if "tokenserver.applications" not in settings: - # Default to just the sync-1.5 application - settings["tokenserver.applications"] = "sync-1.5" - if "tokenserver.secrets.backend" not in settings: - # Default to a single fixed signing secret - settings["tokenserver.secrets.backend"] = "mozsvc.secrets.FixedSecrets" - settings["tokenserver.secrets.secrets"] = [secret] - if "tokenserver.allow_new_users" not in settings: - allow_new_users = settings.get("syncserver.allow_new_users") - if allow_new_users is not None: - settings["tokenserver.allow_new_users"] = allow_new_users - if "hawkauth.secrets.backend" not in settings: - # Default to the same secrets backend as the tokenserver - for key in settings.keys(): - if key.startswith("tokenserver.secrets."): - newkey = "hawkauth" + key[len("tokenserver"):] - settings[newkey] = settings[key] - if "storage.backend" not in settings: - # Default to sql syncstorage backend - settings["storage.backend"] = "syncstorage.storage.sql.SQLStorage" - settings["storage.sqluri"] = sqluri - settings["storage.create_tables"] = True - if "browserid.backend" not in settings: - # Default to remote verifier, with base of public_url as only audience - audience = urlunparse(urlparse(public_url)._replace(path="")) - settings["browserid.backend"] = "tokenserver.verifiers.RemoteVerifier" - settings["browserid.audiences"] = audience - if "loggers" not in settings: - # Default to basic logging config. - root_logger = logging.getLogger("") - if not root_logger.handlers: - logging.basicConfig(level=logging.INFO) - - # Include the relevant sub-packages. - config.scan("syncserver") - config.include("syncstorage", route_prefix="/storage") - config.include("tokenserver", route_prefix="/token") - config.include('pyramid_chameleon') - - # Add a top-level explaination view. - # First view, available at http://localhost:6543/ - def page(request): - result = render('page/index.pt', - {'public_url':public_url}, - request=request) - response = Response(result) - return response - config.add_route('page', '/') - config.add_view(page, route_name='page') - - www = static_view( - os.path.realpath(os.path.dirname(__file__)+"/page/"), - use_subpath=True - ) - # Documentation for Hybrid routing can be found here - # http://docs.pylonsproject.org/projects/pyramid/en/1.0-branch/narr/hybrid.html#using-subpath-in-a-route-pattern - config.add_route('index', '/*subpath', 'www') # subpath is a reserved word - config.add_view(www, route_name='index') - - -@subscriber(NewRequest) -def reconcile_wsgi_environ_with_public_url(event): - """Event-listener that checks and tweaks WSGI environ based on public_url. - - This is a simple trick to help ensure that the configured public_url - matches the actual deployed address. It fixes fixes parts of the WSGI - environ where it makes sense (e.g. SCRIPT_NAME) and warns about any parts - that seem obviously mis-configured (e.g. http:// versus https://). - - It's very important to get public_url and WSGI environ matching exactly, - since they're used for browserid audience checking and HAWK signature - validation, so mismatches can easily cause strange and cryptic errors. - """ - request = event.request - public_url = request.registry.settings["syncserver.public_url"] - p_public_url = urlparse(public_url) - # If we don't have a SCRIPT_NAME, take it from the public_url. - # This is often the case if we're behind e.g. an nginx proxy that - # is serving us at some sub-path. - if not request.script_name: - request.script_name = p_public_url.path.rstrip("/") - # If the environ does not match public_url, requests are almost certainly - # going to fail due to auth errors. We can either bail out early, or we - # can forcibly clobber the WSGI environ with the values from public_url. - # This is a security risk if you've e.g. mis-configured the server, so - # it's not enabled by default. - application_url = request.application_url - if public_url != application_url: - if not request.registry.settings.get("syncserver.force_wsgi_environ"): - msg = "\n".join(( - "The public_url setting doesn't match the application url.", - "This will almost certainly cause authentication failures!", - " public_url setting is: %s" % (public_url,), - " application url is: %s" % (application_url,), - "You can disable this check by setting the force_wsgi_environ", - "option in your config file, but do so at your own risk.", - )) - logger.error(msg) - raise _JSONError([msg], status_code=500) - request.scheme = p_public_url.scheme - request.host = p_public_url.netloc - request.script_name = p_public_url.path.rstrip("/") - - -def get_configurator(global_config, **settings): - """Load a SyncStorge configurator object from deployment settings.""" - config = mozsvc.config.get_configurator(global_config, **settings) - config.begin() - try: - config.include(includeme) - finally: - config.end() - return config - - -def main(global_config, **settings): - """Load a SyncStorage WSGI app from deployment settings.""" - config = get_configurator(global_config, **settings) - return config.make_wsgi_app() diff --git a/sources/syncserver/staticnode.py b/sources/syncserver/staticnode.py deleted file mode 100644 index 560d3ac..0000000 --- a/sources/syncserver/staticnode.py +++ /dev/null @@ -1,221 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Simple node-assignment backend using a single, static node. - -This is a greatly-simplified node-assignment backend. It keeps user records -in an SQL database, but does not attempt to do any node management. All users -are implicitly assigned to a single, static node. - -XXX TODO: move this into the tokenserver repo. - -""" -import time -import urlparse -from mozsvc.exceptions import BackendError - -from sqlalchemy import Column, Integer, String, BigInteger, Index -from sqlalchemy import create_engine, Table, MetaData -from sqlalchemy.pool import QueuePool -from sqlalchemy.sql import text as sqltext -from sqlalchemy.exc import IntegrityError - -from tokenserver.assignment import INodeAssignment -from zope.interface import implements - - -metadata = MetaData() - - -users = Table( - "users", - metadata, - Column("uid", Integer(), primary_key=True, autoincrement=True, - nullable=False), - Column("service", String(32), nullable=False), - Column("email", String(255), nullable=False), - Column("generation", BigInteger(), nullable=False), - Column("client_state", String(32), nullable=False), - Column("created_at", BigInteger(), nullable=False), - Column("replaced_at", BigInteger(), nullable=True), - Index('lookup_idx', 'email', 'service', 'created_at'), - Index('clientstate_idx', 'email', 'service', 'client_state', unique=True), -) - - -_GET_USER_RECORDS = sqltext("""\ -select - uid, generation, client_state -from - users -where - email = :email -and - service = :service -order by - created_at desc, uid desc -limit - 20 -""") - - -_CREATE_USER_RECORD = sqltext("""\ -insert into - users - (service, email, generation, client_state, created_at, replaced_at) -values - (:service, :email, :generation, :client_state, :timestamp, NULL) -""") - - -_UPDATE_GENERATION_NUMBER = sqltext("""\ -update - users -set - generation = :generation -where - service = :service and email = :email and - generation < :generation and replaced_at is null -""") - - -_REPLACE_USER_RECORDS = sqltext("""\ -update - users -set - replaced_at = :timestamp -where - service = :service and email = :email - and replaced_at is null and created_at < :timestamp -""") - - -def get_timestamp(): - """Get current timestamp in milliseconds.""" - return int(time.time() * 1000) - - -class StaticNodeAssignment(object): - implements(INodeAssignment) - - def __init__(self, sqluri, node_url, **kw): - self.sqluri = sqluri - self.node_url = node_url - self.driver = urlparse.urlparse(sqluri).scheme.lower() - sqlkw = { - "logging_name": "syncserver", - "connect_args": {}, - "poolclass": QueuePool, - "pool_reset_on_return": True, - } - if self.driver == "sqlite": - # We must mark it as safe to share sqlite connections between - # threads. The pool will ensure there's on race conditions. - sqlkw["connect_args"]["check_same_thread"] = False - # If using a :memory: database, we must use a QueuePool of size - # 1 so that a single connection is shared by all threads. - if urlparse.urlparse(sqluri).path.lower() in ("/", "/:memory:"): - sqlkw["pool_size"] = 1 - sqlkw["max_overflow"] = 0 - if "mysql" in self.driver: - # Guard against the db closing idle conections. - sqlkw["pool_recycle"] = 3600 - self._engine = create_engine(sqluri, **sqlkw) - users.create(self._engine, checkfirst=True) - - def get_user(self, service, email): - params = {'service': service, 'email': email} - res = self._engine.execute(_GET_USER_RECORDS, **params) - try: - row = res.fetchone() - if row is None: - return None - # The first row is the most up-to-date user record. - user = { - 'email': email, - 'uid': row.uid, - 'node': self.node_url, - 'generation': row.generation, - 'client_state': row.client_state, - 'old_client_states': {} - } - # Any subsequent rows are due to old client-state values. - row = res.fetchone() - while row is not None: - user['old_client_states'][row.client_state] = True - row = res.fetchone() - return user - finally: - res.close() - - def allocate_user(self, service, email, generation=0, client_state=''): - params = { - 'service': service, 'email': email, 'generation': generation, - 'client_state': client_state, 'timestamp': get_timestamp() - } - try: - res = self._engine.execute(_CREATE_USER_RECORD, **params) - except IntegrityError: - raise - return self.get_user(service, email) - else: - res.close() - return { - 'email': email, - 'uid': res.lastrowid, - 'node': self.node_url, - 'generation': generation, - 'client_state': client_state, - 'old_client_states': {} - } - - def update_user(self, service, user, generation=None, client_state=None): - if client_state is None: - # uid can stay the same, just update the generation number. - if generation is not None: - params = { - 'service': service, - 'email': user['email'], - 'generation': generation, - } - res = self._engine.execute(_UPDATE_GENERATION_NUMBER, **params) - res.close() - user['generation'] = max(generation, user['generation']) - else: - # reject previously-seen client-state strings. - if client_state == user['client_state']: - raise BackendError('previously seen client-state string') - if client_state in user['old_client_states']: - raise BackendError('previously seen client-state string') - # need to create a new record for new client_state. - if generation is not None: - generation = max(user['generation'], generation) - else: - generation = user['generation'] - now = get_timestamp() - params = { - 'service': service, 'email': user['email'], - 'generation': generation, 'client_state': client_state, - 'timestamp': now, - } - try: - res = self._engine.execute(_CREATE_USER_RECORD, **params) - except IntegrityError: - user.update(self.get_user(service, user['email'])) - else: - self.get_user(service, user['email']) - user['uid'] = res.lastrowid - user['generation'] = generation - user['old_client_states'][user['client_state']] = True - user['client_state'] = client_state - res.close() - # mark old records as having been replaced. - # if we crash here, they are unmarked and we may fail to - # garbage collect them for a while, but the active state - # will be undamaged. - params = { - 'service': service, 'email': user['email'], 'timestamp': now - } - res = self._engine.execute(_REPLACE_USER_RECORDS, **params) - res.close() diff --git a/sources/syncserver/tests.ini b/sources/syncserver/tests.ini deleted file mode 100644 index fc9ea4b..0000000 --- a/sources/syncserver/tests.ini +++ /dev/null @@ -1,19 +0,0 @@ -[server:main] -use = egg:gunicorn -host = 0.0.0.0 -port = 5000 -workers = 1 -timeout = 30 - -[app:main] -use = egg:SyncServer - -[syncserver] -# This must be edited to point to the public URL of your server. -public_url = http://localhost:5000/ - -# This defines the database in which to store all server data. -#sqluri = sqlite:////tmp/syncserver.db - -# This is a secret key used for signing authentication tokens. -#secret = INSERT_SECRET_KEY_HERE