From cb484b136f8d2115361d90a96a7da2472d2ef621 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Mon, 8 Feb 2016 11:04:35 +0100 Subject: Renamed misc dir to doc --- doc/gn_installation_notes.txt | 346 ++++++++++++++++++++++++++++++++++++++++++ doc/new_variable_names.txt | 6 + doc/notes.txt | 311 +++++++++++++++++++++++++++++++++++++ doc/notes_DA.txt | 10 ++ doc/requirements.txt | 36 +++++ doc/todo.txt | 2 + 6 files changed, 711 insertions(+) create mode 100644 doc/gn_installation_notes.txt create mode 100644 doc/new_variable_names.txt create mode 100644 doc/notes.txt create mode 100644 doc/notes_DA.txt create mode 100644 doc/requirements.txt create mode 100644 doc/todo.txt (limited to 'doc') diff --git a/doc/gn_installation_notes.txt b/doc/gn_installation_notes.txt new file mode 100644 index 00000000..584080f7 --- /dev/null +++ b/doc/gn_installation_notes.txt @@ -0,0 +1,346 @@ +UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration + +Enable UFW: +sudo ufw enable +Check status of UFW: +sudo ufw status verbose + +Limit ssh: +sudo ufw limit ssh + +Enable http, https: +sudo ufw allow http +sudo ufw allow https + +=========================================== + +http://evilrouters.net/2011/02/18/using-etckeeper-with-git-on-ubuntu/ + +Install etckeeper: +sudo apt-get install etckeeper + +Edit /etc/etckeeper/etckeeper.conf to comment out VCS="bzr" and comment in VCS="git" + +Initialize repository: +sudo etckeeper init + +Do initial commit: +sudo etckeeper commit "Initial commit." + +=========================================== + +Get git: + +sudo apt-get install git + +=========================================== + +Clone code from git repository: +git clone http://github.com/zsloan/genenetwork.git gene + +Pull from branch in git repository: +git pull origin flask(or whatever the branch is) + +**"apt-cache search" searches for text in the description for various packages, while +"apt-file search" searches for files within the packages** + +Search for package with a specified file that can be installed with apt-get +apt-file search _______ + +============================================ + +Create trash directory: +mkdir ~/.trash + +============================================ + +Using dpkg + +Check which system packages are already installed: +dpkg -l | less + +=========================================== + +Installing virtualenv: +sudo pip install virtualenv + +Tell virtualenv which directory to use: +virtualenv ve27 + +Start up virtual environment: +source ~/ve27/bin/activate + +=========================================== + +Install libmysqlclient-dev (Mysql-Python dependency) +sudo apt-get install libmysqlclient-dev + +Install python-dev (numpy dependency) +sudo apt-get install python-dev + +Install scipy dependencies: +sudo apt-get install libatlas-base-dev gfortran g++ + +=========================================== + +Install pip: +sudo apt-get install python-pip + +REMEMBER TO SOURCE VE BEFORE INSTALLING + +Comment out in requirements.txt: +Reaper +numarray + +Before installing from requirements.txt, install numpy separately: +pip install numpy==1.7.0 (or whatever version we're using) + +Install from requirements.txt (after activating virtualenv): +pip install -r gene/misc/requirements.txt + +=========================================== + +Installing QTL Reaper + +wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb +mv -v qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb qtlreaper-1.1.1.tar.gz +tar xvf qtlreaper-1.1.1.tar.gz (to unzip) +mkdir /home/zas1024/ve27/include/python2.7/Reaper +python setup.py install + +=========================================== + +Installing numarray 1.5.2 + +wget http://downloads.sourceforge.net/project/numpy/Old%20Numarray/1.5.2/numarray-1.5.2.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%2520Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload&ts=1358978306&use_mirror=superb-dca2 +mv -v numarray-1.5.2.tar.gz?r=http:%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%20Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload numarray-1.5.2.tar.gz +tar xvf numarray-1.5.2.tar.gz +sudo python setup.py install + +=========================================== + +Installing nginx + +sudo -s +nginx=stable # use nginx=development for latest development version +echo "deb http://ppa.launchpad.net/nginx/$nginx/ubuntu lucid main" > /etc/apt/sources.list.d/nginx-$nginx-lucid.list +apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C300EE8C +apt-get update +apt-get install nginx + +Create configuration file in ~/gene/wqflask/other_config/nginx_conf/ (modeled off of the others) +Create symbolic link to config file in /etc/nginx/sites-enabled/: +ln -s dir_to_link_to linking_dir + +Run nginx: +sudo /usr/sbin/nginx + +============================================ + +Copy over zach_settings.py to /home/zas1024 directory (or whatever is home directory) + +export WQFLASK_SETTINGS=~/gene/wqflask/cfg/zach_settings.py (or wherever file is located) + +============================================ + +Install MySQL Server + +sudo apt-get install mysql-server + +mkdir /mnt/big +fdisk /dev/sdb +m: for help +n: new partion +... +w: write to table and exit + +Start MySQL server: +service mysql start + +Stop MySQL server: +service mysql stop + +Change root password: +mysql> UPDATE mysql.user SET Password=PASSWORD('your password') WHERE User='root'; + +Setup accounts in MySQL (first need to delete anonymous/non-root accounts): +#; use mysql; +#; select * from user; +#; delete from user where Host!="localhost"; +#; delete from user where User!="root"; +#; update user set Password = Password('yourpassword') where User='root'; +#; GRANT ALL ON *.* TO 'yourusername'@'%' IDENTIFIED BY 'yourpassword'; +#; select * from user; + +Use Percona to optimise my.cfg file: +https://tools.percona.com/wizard + +To check # CPUs: +grep -c processor /proc/cpuinfo + +To check memory, can use htop + +============================================ + +Check RSA key: +ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key + +03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) + +============================================ + +Installing redis server + +In the dls directory: +$ wget http://redis.googlecode.com/files/redis-2.6.14.tar.gz +$ tar xzf redis-2.6.14.tar.gz +$ cd redis-2.6.14 +$ make + +Run redis (from inside redis-2.6.14 directory): +src/redis-server + +For the rest of the installation instructions, refer to: +http://redis.io/topics/quickstart + +To continue installation (from the README): + +In order to install Redis binaries into /usr/local/bin just use: + + % make install + +Make install will just install binaries in your system, but will not configure +init scripts and configuration files in the appropriate place. This is not +needed if you want just to play a bit with Redis, but if you are installing +it the proper way for a production system, we have a script doing this +for Ubuntu and Debian systems: + + % cd utils + % ./install_server + +The script will ask you a few questions and will setup everything you need +to run Redis properly as a background daemon that will start again on +system reboots. + +In the redis config file (/etc/redis/6379.conf) uncomment the following line: +bind 127.0.0.1 + +This is so only the local computer has access + +============================================ + +Check RSA key: +ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key + +03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) + +============================================ + +Using Yolk + +Install Yolk: +pip install yolk + +Check packages installed in this virtual environment: +yolk -l + +Checks packages that have updates available: +yolk -U + +=========================================== + +Installing yaml + +Install libyaml-dev: +sudo apt-get install libyaml-dev + +Install yaml: +pip install pyyaml + +=========================================== + +Install MySQL Client + +To fix error "mysql_config not found" while installing packages with pip: +sudo apt-get install libmysqlclient-dev + +=========================================== + +Installing R + +sudo apt-get install r-base-dev + +=========================================== + +Installing rpy2 + +pip install rpy2 + +=========================================== + +Install Parallel Python (pp) + +wget http://www.parallelpython.com/downloads/pp/pp-1.6.3.tar.gz +tar xvf pp-1.6.3.tar.gz +python setup.py install + +========================================== + +Install requests + +pip install requests + +========================================== + +Install logging_tree + +pip install logging_tree + +========================================== + +Install colordiff (just to make it easier to view differences in files, not necessary for GN) + +sudo apt-get install colordiff + +========================================== + +Install NTP (network time protocol) + +sudo apt-get install ntp + +========================================== + +To get server running: + +!If having seemingly inexplicable problems with imports, make sure I've started the environment! + +Start up virtual environment: +source ~/ve27/bin/activate============== + +Install requests + +pip install requests + +======================= + +To set WQFLASK_SETTINGS environment variable: +export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) + +To change screen environment variable (if man not working or to get color, for example): +export TERM=screen + +To search for commands in history if necessary: +history | grep "(whatever is being searched for)" + +Run web server: +/usr/local/nginx/sbin/nginx + +Run server: +python runserver.py + +=========================================== + +To do full upgrade (as opposed to apt-get upgrade) +sudo aptitude full-upgrade + +=========================================== \ No newline at end of file diff --git a/doc/new_variable_names.txt b/doc/new_variable_names.txt new file mode 100644 index 00000000..c11c160e --- /dev/null +++ b/doc/new_variable_names.txt @@ -0,0 +1,6 @@ +RISet/riset -> group +webqtlDataset.py -> data_set.py +webqtlDataset (class object) -> DataSet +database/db -> dataset/data_set +DataEditingPage -> show_trait.py/show_trait.html +webqtlTrait -> GeneralTrait \ No newline at end of file diff --git a/doc/notes.txt b/doc/notes.txt new file mode 100644 index 00000000..f8ce2759 --- /dev/null +++ b/doc/notes.txt @@ -0,0 +1,311 @@ +Clone code from git repository: +git clone http://github.com/zsloan/genenetwork.git gene + +Pull from branch in git repository: +git pull origin flask(or whatever the branch is) + +Add all files in current directory branch to be staged to be committed +git add -A + +Git push to my github repository +git push https://github.com/zsloan/genenetwork.git master + +Get the default path being used when a command is typed: +which _______ + +Change default path python looks at when importing +export PYTHONPATH="/home/zas1024/gene/wqflask" + +Get the paths checked when a user types a command: +echo $PATH + +Get all environment variables: +set + + +**** apt-get is an Ubuntu system-wide package manager, while pip is a Python-only system +for python packages **** + +Upgrade system packages for security updates, etc: +apt-get update +apt-get dist-upgrade (can probably be used in place of apt-get upgrade) +apt-get upgrade + +**"apt-cache search" searches for text in the description for various packages, while +"apt-file search" searches for files within the packages** + +Show information about given result item returned by apt-cache search: +apt-cache show _______ + +Search for package with a specified file that can be installed with apt-get +apt-file search _______ + +============================================ + +Check which system packages are installed: +dpkg -l | less + +============================================ + +NEVER USE SUDO WITH PIP + +To fix error "mysql_config not found": +sudo apt-get install libmysqlclient-dev + +Create requirements.txt file: +pip freeze > requirements.txt (or whatever filename) + +Install from requirements.txt: +pip install -r gene/wqflask/requirements.txt -t ve27 + +============================================ + +To get server running: + +!If having seemingly inexplicable problems with imports, make sure I've started the environment! + +Note: Virtual environment is not activated when running as root + +Start up virtual environment: +source ~/ve27/bin/activate + +Deactivate virtual environment: +deactivate (just type deactivate) + +To set WQFLASK_SETTINGS environment variable: +export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) + +zach_settings.py is the local config file with the DB_URI +default_settings.py is the config file with the configuration information that would apply +to any running instance of GeneNetwork + +To change screen environment variable (if man not working or to get color, for example): +export TERM=screen + +To search for commands in history if necessary: +history | grep "(whatever is being searched for)" + +Reload web server: +/usr/sbin/nginx -s reload + +Run server: +python runserver.py + +Run sendmail.py +python send_mail.py + +=========================================== + +UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration + +Enable UFW: +sudo ufw enable +Check status of UFW: +sudo ufw status verbose + +=========================================== + +Update user password: +use mysql; +UPDATE user SET password=PASSWORD('whatever') WHERE user='webqtl'; +FLUSH PRIVILEGES; + +=========================================== + +Redis things + +Restart redis server: +sudo /etc/init.d/redis_6379 stop +sudo /etc/init.d/redis_6379 start + +=========================================== + +Run pylint: +pylint file_name + +For options: +pylint --help + +=========================================== + +Start screen session: + +If "no option -R" or running for first time: +byobu-select-backend +2. screen + +byobu -RD (to start) +control-a then :multiuser on +control-a then :acladd sam + +More info on sharing byobu screen with other users: +http://wiki.networksecuritytoolkit.org/nstwiki/index.php/HowTo_Share_A_Terminal_Session_Using_Screen + +control-a c to create channel + +type: screen -list for sessions +screen -r zas1024/25679.byobu + +or if only one: + +screen -r zas1024/ + +=========================================== + +Start up log: +Go to /tmp and tail -f flask_gn_log + +=========================================== + +Coffeescript Stuff: + +coffee -c (filename) +coffee -c -w . (to watch for changes and recompile in current directory; the "." is for current directory) +coffee --help (for information about setting options) + +=========================================== + +Unset ASKPASS when trying to git push + +unset SSH_ASKPASS + +=========================================== + +htop: Gives information on processes, cpu/memory load, etc +dstat: Also gives various system information, resource usage, etc +df: Reports file system disk space usage + +=========================================== + +ll h*: Finds items in the directory starting with h + +=========================================== + +du -hms * | sort -n : Gives size used by different directories +-h: human readable +-m: in megabytes (default) +-s: summarize + +=========================================== + +rm -rfv + +=========================================== + +cp -a (archive; copies recursively and doesn't follow symbol links) + -i (interactive, prompts before overwrite) + -v (verbose) + -b (backup) + +mv (same as above, but with no -a) + +=========================================== + +Add user: +sudo adduser +Edit /etc/sudoers to give user root privileges + + +=========================================== + +tidyp - Improves/beautifies html code +tidyp -m -i -w 100 index_page.html + +=========================================== + +ps ax - View processes +ps ux (the "u" is for user) + +kill (process #) + +=========================================== + +Installing QTL Reaper + +wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb +tar xvf qtlreaper-1.1.1.tar.gz (to unzip) +python setup.py build +python setup.py install + +=========================================== + +Using Yolk + +Install Yolk: +pip install yolk + +Check packages installed in this virtual environment: +yolk -l + +Checks packages that have updates available: +yolk -U + +=========================================== + +Python stuff: + +Classes should always inherit "object" +To iterate through dictionary items: for X, Y in MyDictionary.items(): + +=========================================== + +Change import path from command line +export PYTHONPATH=~/gene/wqflask (or whatever) + +=========================================== + +Create symbolic link: +ln -s dir_to_link_to linking_dir + +=========================================== + +Reboot server +sudo reboot + +=========================================== + +Using grep: + +grep -ir (search string) (directory) + +-i ignores case +-r searches recursively through subdirectories of directory you specify + +=========================================== + +Command line arguments: + +Use argparse to deal with command line arguments (instead of argv or optparse) + +=========================================== + +Change owner/group: + +chown zas1024 somefile (change owner of somefile to zas1024) +chown zas1024: somefile (change owner and group of somefile to zas1024) +sudo chown root: somefile (change owner/group of somefile to root) + +Change permissions: + +chmod g-wr (remove write/read privileges for group) +chmod g+w (add write privileges for group) +chmod u+rwx (add read, write, and execute privileges for user) +etc. + +=========================================== + +MySQL Things - + +Create fulltext (MATCH AGAINST syntax) index: +CREATE FULLTEXT INDEX __________ ON Table (Column); + +If <4 character searches not working: +Add ft_min_word_len = 2 (to make te minimum characters 2) to config file +Rebuild relevant tables with fulltext indexes with "REPAIR TABLE _______ QUICK" + +=========================================== + +To delete all .pyc files (for example if there's some issue you're having trouble resolving): +find . -name '*.pyc' -delete +http://stackoverflow.com/questions/785519/how-do-i-remove-all-pyc-files-from-a-project + diff --git a/doc/notes_DA.txt b/doc/notes_DA.txt new file mode 100644 index 00000000..410e0182 --- /dev/null +++ b/doc/notes_DA.txt @@ -0,0 +1,10 @@ +Danny's notes about the genenetwork source + +Location of static files: + +Location of HTML templates: wqflask/wqflask/templates/ + +Entry point of the wqflask app: wqflask/wqflask/__init__.py + +Application routes: wqflask/wqflask/views.py + diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 00000000..39ee5652 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,36 @@ +BeautifulSoup==3.2.1 +Flask==0.9 +Flask-Login==0.1.3 +Flask-Mail==0.7.6 +Flask-Principal==0.3.4 +Flask-SQLAlchemy==0.16 +Flask-Security==1.6.0 +Flask-WTF==0.8.3 +Jinja2==2.6 +MySQL-python==1.2.4 +PyYAML==3.10 +#Reaper==1.0 +Reindent==0.1.1 +SQLAlchemy==0.8.0 +WTForms==1.0.3 +Werkzeug==0.8.3 +apache-libcloud==0.12.3 +argparse==1.2.1 +blinker==1.2 +cairosvg==1.0.15 +itsdangerous==0.17 +logging-tree==1.2 +logilab-astng==0.24.3 +logilab-common==0.59.1 +#numarray==1.5.2 +numpy==1.7.0 +passlib==1.6.1 +pp==1.6.3 +pylint==0.27.0 +redis==2.7.2 +requests==1.1.0 +scipy==0.11.0 +simplejson==3.0.7 +wsgiref==0.1.2 +yolk==0.4.3 +XlsxWriter==0.7.2 diff --git a/doc/todo.txt b/doc/todo.txt new file mode 100644 index 00000000..1d781b13 --- /dev/null +++ b/doc/todo.txt @@ -0,0 +1,2 @@ +- Ask Rob about potentially recoding qtlreaper +- Ask Rob about Probe/cellid traits \ No newline at end of file -- cgit v1.2.3 From 7705462a70d17f65bf34a158ea76f100b56582c8 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Mon, 8 Feb 2016 11:22:45 +0100 Subject: Guix: installation notes --- doc/GUIX-Reproducible-from-source.org | 35 +++++++++++++++++++++++++++++++++++ doc/README.org | 28 ++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 doc/GUIX-Reproducible-from-source.org create mode 100644 doc/README.org (limited to 'doc') diff --git a/doc/GUIX-Reproducible-from-source.org b/doc/GUIX-Reproducible-from-source.org new file mode 100644 index 00000000..b88eb9e8 --- /dev/null +++ b/doc/GUIX-Reproducible-from-source.org @@ -0,0 +1,35 @@ +#+TITLE: Installing GeneNetwork services with GNU Guix + +* Table of Contents :TOC: + - [[#introduction][Introduction]] + - [[#binary-deployment][Binary deployment]] + - [[#from-source-deployment][From source deployment]] + +* Introduction + +Large system deployments tend to get very complex. In this document we +explain the GeneNetwork deployment system which is based on GNU Guix +(see Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). + +* Binary deployment + +NYA (will go to README) + +* From source deployment + +GNU Guix allows for [[https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org][reproducible deployment]] based on a checked out +Guix repository - use gn-stable for that: + +#+begin_src sh :lang bash +mkdir genenetwork +cd genenetwork +git checkout https://github.com/genenetwork/guix.git gn-stable-guix +git checkout https://github.com/genenetwork/guix-bioinformatics.git +#+end_src + +Next build guix (and run) following the instructions in [[https://github.com/pjotrp/guix-notes/blob/master/INSTALL.org#building-gnu-guix-from-source-using-guix][guix-notes]]. + +Once that is done we can add the guix-bioinformatics path with + +: env GUIX_PACKAGE_PATH=../guix-bioinformatics ./pre-inst-env guix package -A slurm + diff --git a/doc/README.org b/doc/README.org new file mode 100644 index 00000000..f6ab6a52 --- /dev/null +++ b/doc/README.org @@ -0,0 +1,28 @@ +#+TITLE: Installing GeneNetwork services with GNU Guix + +* Table of Contents :TOC: + - [[#introduction][Introduction]] + - [[#binary-deployment][Binary deployment]] + - [[#from-source-deployment][From source deployment]] + +* Introduction + +Large system deployments tend to get very complex. In this document we +explain the GeneNetwork deployment system which is based on GNU Guix +(see Pjotr's [[https://github.com/pjotrp/guix-notes/blob/master/README.md][Guix-notes]]). + +* Binary deployment + +NYA + +* From source deployment + +GNU Guix allows for [[https://github.com/pjotrp/guix-notes/blob/master/REPRODUCIBLE.org][reproducible deployment]] based on a checked out +Guix repository - use gn-stable for that: + +#+begin_src sh :lang bash +mkdir genenetwork +cd genenetwork +git checkout https://github.com/genenetwork/guix.git gn-stable-guix +git checkout https://github.com/genenetwork/guix-bioinformatics.git +#+end_src -- cgit v1.2.3