From ff679636d3ee9ae2388fab3fe5d091ef2f00a8e3 Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Sat, 30 Dec 2023 09:27:49 +0000 Subject: Move default settings into gn2 module. --- bin/genenetwork2 | 14 ++-- bin/test-website | 2 +- doc/GUIX-Reproducible-from-source.org | 4 +- doc/development.org | 2 +- doc/testing.org | 4 +- etc/default_settings.py | 122 ---------------------------------- gn2/default_settings.py | 122 ++++++++++++++++++++++++++++++++++ gn2/run_gunicorn.py | 2 +- gn2/utility/tools.py | 2 +- gn2/wqflask/__init__.py | 4 +- scripts/performance/README.org | 2 +- scripts/run_debug.sh | 2 +- test/requests/test-website.py | 2 +- 13 files changed, 141 insertions(+), 143 deletions(-) delete mode 100644 etc/default_settings.py create mode 100644 gn2/default_settings.py diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 2ac83c7f..3e2f31d3 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -25,15 +25,15 @@ # webserver) run from the base-dir with settings file and add that # script with a -c switch, e.g. # -# env GN2_PROFILE=/usr/local/guix-profiles/gn-latest-20190905 TMPDIR=/export/local/home/zas1024/gn2-zach/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/local/home/zas1024/gn2-zach/genotype_files SQL_URI=mysql://webqtlout:webqtlout@lily.uthsc.edu/db_webqtl ./bin/genenetwork2 ./etc/default_settings.py -c ./maintenance/gen_select_dataset.py +# env GN2_PROFILE=/usr/local/guix-profiles/gn-latest-20190905 TMPDIR=/export/local/home/zas1024/gn2-zach/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/local/home/zas1024/gn2-zach/genotype_files SQL_URI=mysql://webqtlout:webqtlout@lily.uthsc.edu/db_webqtl ./bin/genenetwork2 ./gn2/default_settings.py -c ./maintenance/gen_select_dataset.py # # To run any script in the environment # -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD" +# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./gn2/default_settings.py -cli echo "HELLO WORLD" # # To get a python REPL(!) # -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli python +# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./gn2/default_settings.py -cli python # # For development you may want to run # @@ -42,15 +42,15 @@ # For staging and production we use gunicorn. Run with something like # (note you have to provide the server port). Provide a settings file! # -# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-prod +# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./gn2/default_settings.py -gunicorn-prod # # For development use # -# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev +# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./gn2/default_settings.py -gunicorn-dev # # For extra flexibility you can also provide gunicorn parameters yourself with something like # -# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" +# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./gn2/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" SCRIPT=$(realpath "$0") echo SCRIPT="${SCRIPT}" @@ -77,7 +77,7 @@ fi settings=$1 if [ -z "${settings}" ]; then - settings=$GN2_BASE_DIR/etc/default_settings.py + settings=$GN2_BASE_DIR/gn2/default_settings.py else shift fi diff --git a/bin/test-website b/bin/test-website index 7fbcfd2f..43264472 100755 --- a/bin/test-website +++ b/bin/test-website @@ -2,6 +2,6 @@ if [ -z $GN2_PROFILE ]; then echo "Run request tests with something like" - echo env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003 + echo env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./gn2/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003 exit 1 fi diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org index fffa9571..50abec3a 100644 --- a/doc/GUIX-Reproducible-from-source.org +++ b/doc/GUIX-Reproducible-from-source.org @@ -252,7 +252,7 @@ Inside the repository: : ./bin/genenetwork2 Will fire up your local repo http://localhost:5003/ using the -settings in ./etc/default_settings.py. These settings may +settings in ./gn2/default_settings.py. These settings may not reflect your system. To override settings create your own from a copy of default_settings.py and pass it into GN2 with @@ -265,7 +265,7 @@ software. If something is not working, take a hint from the settings file that comes in the Guix installation. It sits in something like -: cat ~/.guix-profile/lib/python3.8/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py +: cat ~/.guix-profile/lib/python3.8/site-packages/genenetwork2-2.0-py2.7.egg/gn2/default_settings.py ** Set up nginx port forwarding diff --git a/doc/development.org b/doc/development.org index cd3beea3..4f3693ee 100644 --- a/doc/development.org +++ b/doc/development.org @@ -86,7 +86,7 @@ In fact you can kick off a Python shell with something like env SERVER_PORT=5013 WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ SQL_URI=mysql://gn2:webqtl@localhost/db_webqtl_s \ GN2_PROFILE=~/opt/genenetwork2 \ - ./bin/genenetwork2 ./etc/default_settings.py -c + ./bin/genenetwork2 ./gn2/default_settings.py -c Python 2.7.17 (default, Jan 1 1970, 00:00:01) [GCC 7.5.0] on linux2 Type "help", "copyright", "credits" or "license" for more information. diff --git a/doc/testing.org b/doc/testing.org index d5ab117d..5c1e954e 100644 --- a/doc/testing.org +++ b/doc/testing.org @@ -35,14 +35,14 @@ Run the tests from the root of the genenetwork2 source tree as. Ensure that Redis and Mariadb are running. To run Mechanical Rob: -: time env GN2_PROFILE=~/opt/genenetwork2 TMPDIR=~/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 ./etc/default_settings.py -c ~/projects/genenetwork2/test/requests/test-website.py -a http://localhost:5004 +: time env GN2_PROFILE=~/opt/genenetwork2 TMPDIR=~/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 ./gn2/default_settings.py -c ~/projects/genenetwork2/test/requests/test-website.py -a http://localhost:5004 Use these aliases for the following examples. #+begin_src sh alias runpython="env GN2_PROFILE=~/opt/gn-latest TMPDIR=/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 -alias runcmd="time env GN2_PROFILE=~/opt/gn-latest TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 ./etc/default_settings.py -cli" +alias runcmd="time env GN2_PROFILE=~/opt/gn-latest TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ ./bin/genenetwork2 ./gn2/default_settings.py -cli" #+end_src You could use them in your =.bashrc= or =.zshrc= file. diff --git a/etc/default_settings.py b/etc/default_settings.py deleted file mode 100644 index c49a3c25..00000000 --- a/etc/default_settings.py +++ /dev/null @@ -1,122 +0,0 @@ -# Default settings file defines a single Flask process for the Python -# webserver running in developer mode with limited console -# output. Copy this file and run it from ./bin/genenetwork2 configfile -# -# Note: these settings are fetched in ./wqflask/utilities/tools.py -# which has support for overriding them through environment variables, -# e.g. -# -# env LOG_SQL=True USE_REDIS=False ./bin/genenetwork2 -# env LOG_LEVEL=DEBUG ./bin/genenetwork2 ~/gn2_settings.py -# -# Typically you need to set GN2_PROFILE too. -# -# Note also that in the near future we will additionally fetch -# settings from a JSON file -# -# Note: values for False and 0 have to be strings here - otherwise -# Flask won't pick them up -# -# For GNU Guix deployment also check the paths in -# -# ~/.guix-profile/lib/python3.8/site-packages/genenetwork2-2.0-py2.7.egg/etc/default_settings.py - -import os -import sys - -GN_VERSION = "2.11-rc2" - -SECRET_KEY = "" - -# Redis -REDIS_URL = "redis://:@localhost:6379/0" - -# gn2-proxy -GN2_PROXY = "http://localhost:8080" - -# GN PROXY -GN_PROXY_URL="https://genenetwork.org/gn3-proxy/" - -# ---- MySQL - -SQL_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s" -SQL_ALCHEMY_POOL_RECYCLE = 3600 -GN_SERVER_URL = "http://localhost:8880/api/" # REST API server -AUTH_SERVER_URL="http://localhost:9094/" -GN2_BASE_URL = "http://genenetwork.org/" # to pick up REST API -GN2_BRANCH_URL = GN2_BASE_URL - -# ---- Flask configuration (see website) -TRAP_BAD_REQUEST_ERRORS = True -SECURITY_CONFIRMABLE = True -SECURITY_TRACKABLE = True -SECURITY_REGISTERABLE = True -SECURITY_RECOVERABLE = True -SECURITY_EMAIL_SENDER = "no-reply@genenetwork.org" -SECURITY_POST_LOGIN_VIEW = "/thank_you" - -SERVER_PORT = 5003 - -SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' - -GITHUB_CLIENT_ID = "UNKNOWN" -GITHUB_CLIENT_SECRET = "UNKNOWN" -GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize" -GITHUB_API_URL = "https://api.github.com/user" - -ORCID_CLIENT_ID = "UNKNOWN" -ORCID_CLIENT_SECRET = "UNKNOWN" -ORCID_AUTH_URL = "https://orcid.org/oauth/authorize" -ORCID_TOKEN_URL = "https://orcid.org/oauth/token" - -ELASTICSEARCH_HOST = "localhost" -ELASTICSEARCH_PORT = '9200' - -SMTP_CONNECT = "localhost" -SMTP_USERNAME = "UNKNOWN" -SMTP_PASSWORD = "UNKNOWN" - - -# ---- Behavioural settings (defaults) note that logger and log levels can -# be overridden at the module level and with enviroment settings -WEBSERVER_MODE = 'DEV' # Python webserver mode (DEBUG|DEV|PROD) -WEBSERVER_BRANDING = None # Set the branding (nyi) -WEBSERVER_DEPLOY = None # Deployment specifics (nyi) -WEBSERVER_URL = "http://localhost:" + str(SERVER_PORT) + "/" # external URL - -LOG_LEVEL = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL) -LOG_LEVEL_DEBUG = '0' # logger.debugf log level (0-5, 5 = show all) -LOG_SQL = 'False' # Log SQL/backend and GN_SERVER calls -LOG_SQL_ALCHEMY = 'False' -LOG_BENCH = True # Log bench marks - -USE_REDIS = True # REDIS caching (note that redis will be phased out) -USE_GN_SERVER = 'False' # Use GN_SERVER SQL calls -HOME = os.environ['HOME'] - -# ---- Default locations -# base dir for all static data files -GENENETWORK_FILES = HOME + "/genotype_files" - -# ---- Path overrides for Genenetwork - the defaults are normally -# picked up from Guix or in the HOME directory - -# TMPDIR is normally picked up from the environment -# PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files (unused) - -# ---- Local path to JS libraries - for development modules (only) -JS_GN_PATH = os.environ['HOME'] + "/genenetwork/javascript" - -# ---- GN2 Executables (overwrite for testing only) -# PLINK_COMMAND = str.strip(os.popen("which plink2").read()) -# GEMMA_COMMAND = str.strip(os.popen("which gemma").read()) -REAPER_COMMAND = os.environ['GN2_PROFILE'] + "/bin/qtlreaper" -CORRELATION_COMMAND = os.environ["GN2_PROFILE"] + "/bin/correlation_rust" -# GEMMA_WRAPPER_COMMAND = str.strip(os.popen("which gemma-wrapper").read()) - -OAUTH2_CLIENT_ID="0bbfca82-d73f-4bd4-a140-5ae7abb4a64d" -OAUTH2_CLIENT_SECRET="yadabadaboo" - -SESSION_TYPE = "redis" -SESSION_PERMANENT = True -SESSION_USE_SIGNER = True diff --git a/gn2/default_settings.py b/gn2/default_settings.py new file mode 100644 index 00000000..e781f196 --- /dev/null +++ b/gn2/default_settings.py @@ -0,0 +1,122 @@ +# Default settings file defines a single Flask process for the Python +# webserver running in developer mode with limited console +# output. Copy this file and run it from ./bin/genenetwork2 configfile +# +# Note: these settings are fetched in ./wqflask/utilities/tools.py +# which has support for overriding them through environment variables, +# e.g. +# +# env LOG_SQL=True USE_REDIS=False ./bin/genenetwork2 +# env LOG_LEVEL=DEBUG ./bin/genenetwork2 ~/gn2_settings.py +# +# Typically you need to set GN2_PROFILE too. +# +# Note also that in the near future we will additionally fetch +# settings from a JSON file +# +# Note: values for False and 0 have to be strings here - otherwise +# Flask won't pick them up +# +# For GNU Guix deployment also check the paths in +# +# ~/.guix-profile/lib/python3.8/site-packages/genenetwork2-2.0-py2.7.egg/gn2/default_settings.py + +import os +import sys + +GN_VERSION = "2.11-rc2" + +SECRET_KEY = "" + +# Redis +REDIS_URL = "redis://:@localhost:6379/0" + +# gn2-proxy +GN2_PROXY = "http://localhost:8080" + +# GN PROXY +GN_PROXY_URL="https://genenetwork.org/gn3-proxy/" + +# ---- MySQL + +SQL_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s" +SQL_ALCHEMY_POOL_RECYCLE = 3600 +GN_SERVER_URL = "http://localhost:8880/api/" # REST API server +AUTH_SERVER_URL="http://localhost:9094/" +GN2_BASE_URL = "http://genenetwork.org/" # to pick up REST API +GN2_BRANCH_URL = GN2_BASE_URL + +# ---- Flask configuration (see website) +TRAP_BAD_REQUEST_ERRORS = True +SECURITY_CONFIRMABLE = True +SECURITY_TRACKABLE = True +SECURITY_REGISTERABLE = True +SECURITY_RECOVERABLE = True +SECURITY_EMAIL_SENDER = "no-reply@genenetwork.org" +SECURITY_POST_LOGIN_VIEW = "/thank_you" + +SERVER_PORT = 5003 + +SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' + +GITHUB_CLIENT_ID = "UNKNOWN" +GITHUB_CLIENT_SECRET = "UNKNOWN" +GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize" +GITHUB_API_URL = "https://api.github.com/user" + +ORCID_CLIENT_ID = "UNKNOWN" +ORCID_CLIENT_SECRET = "UNKNOWN" +ORCID_AUTH_URL = "https://orcid.org/oauth/authorize" +ORCID_TOKEN_URL = "https://orcid.org/oauth/token" + +ELASTICSEARCH_HOST = "localhost" +ELASTICSEARCH_PORT = '9200' + +SMTP_CONNECT = "localhost" +SMTP_USERNAME = "UNKNOWN" +SMTP_PASSWORD = "UNKNOWN" + + +# ---- Behavioural settings (defaults) note that logger and log levels can +# be overridden at the module level and with enviroment settings +WEBSERVER_MODE = 'DEV' # Python webserver mode (DEBUG|DEV|PROD) +WEBSERVER_BRANDING = None # Set the branding (nyi) +WEBSERVER_DEPLOY = None # Deployment specifics (nyi) +WEBSERVER_URL = "http://localhost:" + str(SERVER_PORT) + "/" # external URL + +LOG_LEVEL = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL) +LOG_LEVEL_DEBUG = '0' # logger.debugf log level (0-5, 5 = show all) +LOG_SQL = 'False' # Log SQL/backend and GN_SERVER calls +LOG_SQL_ALCHEMY = 'False' +LOG_BENCH = True # Log bench marks + +USE_REDIS = True # REDIS caching (note that redis will be phased out) +USE_GN_SERVER = 'False' # Use GN_SERVER SQL calls +HOME = os.environ['HOME'] + +# ---- Default locations +# base dir for all static data files +GENENETWORK_FILES = HOME + "/genotype_files" + +# ---- Path overrides for Genenetwork - the defaults are normally +# picked up from Guix or in the HOME directory + +# TMPDIR is normally picked up from the environment +# PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files (unused) + +# ---- Local path to JS libraries - for development modules (only) +JS_GN_PATH = os.environ['HOME'] + "/genenetwork/javascript" + +# ---- GN2 Executables (overwrite for testing only) +# PLINK_COMMAND = str.strip(os.popen("which plink2").read()) +# GEMMA_COMMAND = str.strip(os.popen("which gemma").read()) +REAPER_COMMAND = os.environ['GN2_PROFILE'] + "/bin/qtlreaper" +CORRELATION_COMMAND = os.environ["GN2_PROFILE"] + "/bin/correlation_rust" +# GEMMA_WRAPPER_COMMAND = str.strip(os.popen("which gemma-wrapper").read()) + +OAUTH2_CLIENT_ID="0bbfca82-d73f-4bd4-a140-5ae7abb4a64d" +OAUTH2_CLIENT_SECRET="yadabadaboo" + +SESSION_TYPE = "redis" +SESSION_PERMANENT = True +SESSION_USE_SIGNER = True diff --git a/gn2/run_gunicorn.py b/gn2/run_gunicorn.py index 6de470fe..370d2425 100644 --- a/gn2/run_gunicorn.py +++ b/gn2/run_gunicorn.py @@ -2,7 +2,7 @@ # # Run standalone with # -# ./bin/genenetwork2 ./etc/default_settings.py -c run_gunicorn.py +# ./bin/genenetwork2 ./gn2/default_settings.py -c run_gunicorn.py # from flask import Flask # application = Flask(__name__) diff --git a/gn2/utility/tools.py b/gn2/utility/tools.py index 159643d3..29d94a43 100644 --- a/gn2/utility/tools.py +++ b/gn2/utility/tools.py @@ -40,7 +40,7 @@ def get_setting(command_id, guess=None): Note that we do not use the system path. This is on purpose because it will mess up controlled (reproducible) deployment. The proper way is to either use the GNU Guix defaults as listed in - etc/default_settings.py or override them yourself by creating a + gn2/default_settings.py or override them yourself by creating a different settings.py file (or setting the environment). """ diff --git a/gn2/wqflask/__init__.py b/gn2/wqflask/__init__.py index 9b714868..bca95275 100644 --- a/gn2/wqflask/__init__.py +++ b/gn2/wqflask/__init__.py @@ -48,9 +48,7 @@ app = Flask(__name__) # See http://flask.pocoo.org/docs/config/#configuring-from-files # Note no longer use the badly named WQFLASK_OVERRIDES (nyi) -default_settings_file = Path(Path(__file__).parent.parent.parent, - "etc/default_settings.py") -app.config.from_pyfile(default_settings_file) +app.config.from_object('gn2.default_settings') app.config.from_envvar('GN2_SETTINGS') app.jinja_env.globals.update( diff --git a/scripts/performance/README.org b/scripts/performance/README.org index f7fb392f..0ca5db18 100644 --- a/scripts/performance/README.org +++ b/scripts/performance/README.org @@ -17,7 +17,7 @@ env GN2_PROFILE=$HOME/opt/genenetwork2 \ GN3_LOCAL_URL=http://localhost:8083 \ GN_LOCAL_URL=http://localhost:8083 \ $HOME/projects/oqo-genenetwork2/bin/genenetwork2 \ - $HOME/projects/oqo-genenetwork2/etc/default_settings.py\ + $HOME/projects/oqo-genenetwork2/gn2/default_settings.py\ -cli $* #+end_src diff --git a/scripts/run_debug.sh b/scripts/run_debug.sh index be7ecdb8..bcb63c5b 100755 --- a/scripts/run_debug.sh +++ b/scripts/run_debug.sh @@ -16,5 +16,5 @@ echo Running source from $cwd mkdir -p ~/tmp -env TMPDIR=~/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG GENENETWORK_FILES=/export/data/genenetwork/genotype_files/ SQL_URI=mysql://webqtlout:webqtlout@localhost/db_webqtl ./bin/genenetwork2 etc/default_settings.py -gunicorn-dev +env TMPDIR=~/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG GENENETWORK_FILES=/export/data/genenetwork/genotype_files/ SQL_URI=mysql://webqtlout:webqtlout@localhost/db_webqtl ./bin/genenetwork2 gn2/default_settings.py -gunicorn-dev diff --git a/test/requests/test-website.py b/test/requests/test-website.py index 91bcb12d..5090d465 100755 --- a/test/requests/test-website.py +++ b/test/requests/test-website.py @@ -1,6 +1,6 @@ # Run with something like # -# env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003 +# env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./gn2/default_settings.py -c ../test/requests/test-website.py http://localhost:5003 # # Mostly to pick up the Guix GN2_PROFILE and python modules -- cgit v1.2.3