From cb484b136f8d2115361d90a96a7da2472d2ef621 Mon Sep 17 00:00:00 2001 From: pjotrp Date: Mon, 8 Feb 2016 11:04:35 +0100 Subject: Renamed misc dir to doc --- doc/gn_installation_notes.txt | 346 +++++++++++++++++++++++++++++++++++++++++ doc/new_variable_names.txt | 6 + doc/notes.txt | 311 ++++++++++++++++++++++++++++++++++++ doc/notes_DA.txt | 10 ++ doc/requirements.txt | 36 +++++ doc/todo.txt | 2 + misc/gn_installation_notes.txt | 346 ----------------------------------------- misc/new_variable_names.txt | 6 - misc/notes.txt | 311 ------------------------------------ misc/notes_DA.txt | 10 -- misc/requirements.txt | 36 ----- misc/todo.txt | 2 - 12 files changed, 711 insertions(+), 711 deletions(-) create mode 100644 doc/gn_installation_notes.txt create mode 100644 doc/new_variable_names.txt create mode 100644 doc/notes.txt create mode 100644 doc/notes_DA.txt create mode 100644 doc/requirements.txt create mode 100644 doc/todo.txt delete mode 100644 misc/gn_installation_notes.txt delete mode 100644 misc/new_variable_names.txt delete mode 100644 misc/notes.txt delete mode 100644 misc/notes_DA.txt delete mode 100644 misc/requirements.txt delete mode 100644 misc/todo.txt diff --git a/doc/gn_installation_notes.txt b/doc/gn_installation_notes.txt new file mode 100644 index 00000000..584080f7 --- /dev/null +++ b/doc/gn_installation_notes.txt @@ -0,0 +1,346 @@ +UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration + +Enable UFW: +sudo ufw enable +Check status of UFW: +sudo ufw status verbose + +Limit ssh: +sudo ufw limit ssh + +Enable http, https: +sudo ufw allow http +sudo ufw allow https + +=========================================== + +http://evilrouters.net/2011/02/18/using-etckeeper-with-git-on-ubuntu/ + +Install etckeeper: +sudo apt-get install etckeeper + +Edit /etc/etckeeper/etckeeper.conf to comment out VCS="bzr" and comment in VCS="git" + +Initialize repository: +sudo etckeeper init + +Do initial commit: +sudo etckeeper commit "Initial commit." + +=========================================== + +Get git: + +sudo apt-get install git + +=========================================== + +Clone code from git repository: +git clone http://github.com/zsloan/genenetwork.git gene + +Pull from branch in git repository: +git pull origin flask(or whatever the branch is) + +**"apt-cache search" searches for text in the description for various packages, while +"apt-file search" searches for files within the packages** + +Search for package with a specified file that can be installed with apt-get +apt-file search _______ + +============================================ + +Create trash directory: +mkdir ~/.trash + +============================================ + +Using dpkg + +Check which system packages are already installed: +dpkg -l | less + +=========================================== + +Installing virtualenv: +sudo pip install virtualenv + +Tell virtualenv which directory to use: +virtualenv ve27 + +Start up virtual environment: +source ~/ve27/bin/activate + +=========================================== + +Install libmysqlclient-dev (Mysql-Python dependency) +sudo apt-get install libmysqlclient-dev + +Install python-dev (numpy dependency) +sudo apt-get install python-dev + +Install scipy dependencies: +sudo apt-get install libatlas-base-dev gfortran g++ + +=========================================== + +Install pip: +sudo apt-get install python-pip + +REMEMBER TO SOURCE VE BEFORE INSTALLING + +Comment out in requirements.txt: +Reaper +numarray + +Before installing from requirements.txt, install numpy separately: +pip install numpy==1.7.0 (or whatever version we're using) + +Install from requirements.txt (after activating virtualenv): +pip install -r gene/misc/requirements.txt + +=========================================== + +Installing QTL Reaper + +wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb +mv -v qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb qtlreaper-1.1.1.tar.gz +tar xvf qtlreaper-1.1.1.tar.gz (to unzip) +mkdir /home/zas1024/ve27/include/python2.7/Reaper +python setup.py install + +=========================================== + +Installing numarray 1.5.2 + +wget http://downloads.sourceforge.net/project/numpy/Old%20Numarray/1.5.2/numarray-1.5.2.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%2520Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload&ts=1358978306&use_mirror=superb-dca2 +mv -v numarray-1.5.2.tar.gz?r=http:%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%20Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload numarray-1.5.2.tar.gz +tar xvf numarray-1.5.2.tar.gz +sudo python setup.py install + +=========================================== + +Installing nginx + +sudo -s +nginx=stable # use nginx=development for latest development version +echo "deb http://ppa.launchpad.net/nginx/$nginx/ubuntu lucid main" > /etc/apt/sources.list.d/nginx-$nginx-lucid.list +apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C300EE8C +apt-get update +apt-get install nginx + +Create configuration file in ~/gene/wqflask/other_config/nginx_conf/ (modeled off of the others) +Create symbolic link to config file in /etc/nginx/sites-enabled/: +ln -s dir_to_link_to linking_dir + +Run nginx: +sudo /usr/sbin/nginx + +============================================ + +Copy over zach_settings.py to /home/zas1024 directory (or whatever is home directory) + +export WQFLASK_SETTINGS=~/gene/wqflask/cfg/zach_settings.py (or wherever file is located) + +============================================ + +Install MySQL Server + +sudo apt-get install mysql-server + +mkdir /mnt/big +fdisk /dev/sdb +m: for help +n: new partion +... +w: write to table and exit + +Start MySQL server: +service mysql start + +Stop MySQL server: +service mysql stop + +Change root password: +mysql> UPDATE mysql.user SET Password=PASSWORD('your password') WHERE User='root'; + +Setup accounts in MySQL (first need to delete anonymous/non-root accounts): +#; use mysql; +#; select * from user; +#; delete from user where Host!="localhost"; +#; delete from user where User!="root"; +#; update user set Password = Password('yourpassword') where User='root'; +#; GRANT ALL ON *.* TO 'yourusername'@'%' IDENTIFIED BY 'yourpassword'; +#; select * from user; + +Use Percona to optimise my.cfg file: +https://tools.percona.com/wizard + +To check # CPUs: +grep -c processor /proc/cpuinfo + +To check memory, can use htop + +============================================ + +Check RSA key: +ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key + +03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) + +============================================ + +Installing redis server + +In the dls directory: +$ wget http://redis.googlecode.com/files/redis-2.6.14.tar.gz +$ tar xzf redis-2.6.14.tar.gz +$ cd redis-2.6.14 +$ make + +Run redis (from inside redis-2.6.14 directory): +src/redis-server + +For the rest of the installation instructions, refer to: +http://redis.io/topics/quickstart + +To continue installation (from the README): + +In order to install Redis binaries into /usr/local/bin just use: + + % make install + +Make install will just install binaries in your system, but will not configure +init scripts and configuration files in the appropriate place. This is not +needed if you want just to play a bit with Redis, but if you are installing +it the proper way for a production system, we have a script doing this +for Ubuntu and Debian systems: + + % cd utils + % ./install_server + +The script will ask you a few questions and will setup everything you need +to run Redis properly as a background daemon that will start again on +system reboots. + +In the redis config file (/etc/redis/6379.conf) uncomment the following line: +bind 127.0.0.1 + +This is so only the local computer has access + +============================================ + +Check RSA key: +ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key + +03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) + +============================================ + +Using Yolk + +Install Yolk: +pip install yolk + +Check packages installed in this virtual environment: +yolk -l + +Checks packages that have updates available: +yolk -U + +=========================================== + +Installing yaml + +Install libyaml-dev: +sudo apt-get install libyaml-dev + +Install yaml: +pip install pyyaml + +=========================================== + +Install MySQL Client + +To fix error "mysql_config not found" while installing packages with pip: +sudo apt-get install libmysqlclient-dev + +=========================================== + +Installing R + +sudo apt-get install r-base-dev + +=========================================== + +Installing rpy2 + +pip install rpy2 + +=========================================== + +Install Parallel Python (pp) + +wget http://www.parallelpython.com/downloads/pp/pp-1.6.3.tar.gz +tar xvf pp-1.6.3.tar.gz +python setup.py install + +========================================== + +Install requests + +pip install requests + +========================================== + +Install logging_tree + +pip install logging_tree + +========================================== + +Install colordiff (just to make it easier to view differences in files, not necessary for GN) + +sudo apt-get install colordiff + +========================================== + +Install NTP (network time protocol) + +sudo apt-get install ntp + +========================================== + +To get server running: + +!If having seemingly inexplicable problems with imports, make sure I've started the environment! + +Start up virtual environment: +source ~/ve27/bin/activate============== + +Install requests + +pip install requests + +======================= + +To set WQFLASK_SETTINGS environment variable: +export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) + +To change screen environment variable (if man not working or to get color, for example): +export TERM=screen + +To search for commands in history if necessary: +history | grep "(whatever is being searched for)" + +Run web server: +/usr/local/nginx/sbin/nginx + +Run server: +python runserver.py + +=========================================== + +To do full upgrade (as opposed to apt-get upgrade) +sudo aptitude full-upgrade + +=========================================== \ No newline at end of file diff --git a/doc/new_variable_names.txt b/doc/new_variable_names.txt new file mode 100644 index 00000000..c11c160e --- /dev/null +++ b/doc/new_variable_names.txt @@ -0,0 +1,6 @@ +RISet/riset -> group +webqtlDataset.py -> data_set.py +webqtlDataset (class object) -> DataSet +database/db -> dataset/data_set +DataEditingPage -> show_trait.py/show_trait.html +webqtlTrait -> GeneralTrait \ No newline at end of file diff --git a/doc/notes.txt b/doc/notes.txt new file mode 100644 index 00000000..f8ce2759 --- /dev/null +++ b/doc/notes.txt @@ -0,0 +1,311 @@ +Clone code from git repository: +git clone http://github.com/zsloan/genenetwork.git gene + +Pull from branch in git repository: +git pull origin flask(or whatever the branch is) + +Add all files in current directory branch to be staged to be committed +git add -A + +Git push to my github repository +git push https://github.com/zsloan/genenetwork.git master + +Get the default path being used when a command is typed: +which _______ + +Change default path python looks at when importing +export PYTHONPATH="/home/zas1024/gene/wqflask" + +Get the paths checked when a user types a command: +echo $PATH + +Get all environment variables: +set + + +**** apt-get is an Ubuntu system-wide package manager, while pip is a Python-only system +for python packages **** + +Upgrade system packages for security updates, etc: +apt-get update +apt-get dist-upgrade (can probably be used in place of apt-get upgrade) +apt-get upgrade + +**"apt-cache search" searches for text in the description for various packages, while +"apt-file search" searches for files within the packages** + +Show information about given result item returned by apt-cache search: +apt-cache show _______ + +Search for package with a specified file that can be installed with apt-get +apt-file search _______ + +============================================ + +Check which system packages are installed: +dpkg -l | less + +============================================ + +NEVER USE SUDO WITH PIP + +To fix error "mysql_config not found": +sudo apt-get install libmysqlclient-dev + +Create requirements.txt file: +pip freeze > requirements.txt (or whatever filename) + +Install from requirements.txt: +pip install -r gene/wqflask/requirements.txt -t ve27 + +============================================ + +To get server running: + +!If having seemingly inexplicable problems with imports, make sure I've started the environment! + +Note: Virtual environment is not activated when running as root + +Start up virtual environment: +source ~/ve27/bin/activate + +Deactivate virtual environment: +deactivate (just type deactivate) + +To set WQFLASK_SETTINGS environment variable: +export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) + +zach_settings.py is the local config file with the DB_URI +default_settings.py is the config file with the configuration information that would apply +to any running instance of GeneNetwork + +To change screen environment variable (if man not working or to get color, for example): +export TERM=screen + +To search for commands in history if necessary: +history | grep "(whatever is being searched for)" + +Reload web server: +/usr/sbin/nginx -s reload + +Run server: +python runserver.py + +Run sendmail.py +python send_mail.py + +=========================================== + +UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration + +Enable UFW: +sudo ufw enable +Check status of UFW: +sudo ufw status verbose + +=========================================== + +Update user password: +use mysql; +UPDATE user SET password=PASSWORD('whatever') WHERE user='webqtl'; +FLUSH PRIVILEGES; + +=========================================== + +Redis things + +Restart redis server: +sudo /etc/init.d/redis_6379 stop +sudo /etc/init.d/redis_6379 start + +=========================================== + +Run pylint: +pylint file_name + +For options: +pylint --help + +=========================================== + +Start screen session: + +If "no option -R" or running for first time: +byobu-select-backend +2. screen + +byobu -RD (to start) +control-a then :multiuser on +control-a then :acladd sam + +More info on sharing byobu screen with other users: +http://wiki.networksecuritytoolkit.org/nstwiki/index.php/HowTo_Share_A_Terminal_Session_Using_Screen + +control-a c to create channel + +type: screen -list for sessions +screen -r zas1024/25679.byobu + +or if only one: + +screen -r zas1024/ + +=========================================== + +Start up log: +Go to /tmp and tail -f flask_gn_log + +=========================================== + +Coffeescript Stuff: + +coffee -c (filename) +coffee -c -w . (to watch for changes and recompile in current directory; the "." is for current directory) +coffee --help (for information about setting options) + +=========================================== + +Unset ASKPASS when trying to git push + +unset SSH_ASKPASS + +=========================================== + +htop: Gives information on processes, cpu/memory load, etc +dstat: Also gives various system information, resource usage, etc +df: Reports file system disk space usage + +=========================================== + +ll h*: Finds items in the directory starting with h + +=========================================== + +du -hms * | sort -n : Gives size used by different directories +-h: human readable +-m: in megabytes (default) +-s: summarize + +=========================================== + +rm -rfv + +=========================================== + +cp -a (archive; copies recursively and doesn't follow symbol links) + -i (interactive, prompts before overwrite) + -v (verbose) + -b (backup) + +mv (same as above, but with no -a) + +=========================================== + +Add user: +sudo adduser +Edit /etc/sudoers to give user root privileges + + +=========================================== + +tidyp - Improves/beautifies html code +tidyp -m -i -w 100 index_page.html + +=========================================== + +ps ax - View processes +ps ux (the "u" is for user) + +kill (process #) + +=========================================== + +Installing QTL Reaper + +wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb +tar xvf qtlreaper-1.1.1.tar.gz (to unzip) +python setup.py build +python setup.py install + +=========================================== + +Using Yolk + +Install Yolk: +pip install yolk + +Check packages installed in this virtual environment: +yolk -l + +Checks packages that have updates available: +yolk -U + +=========================================== + +Python stuff: + +Classes should always inherit "object" +To iterate through dictionary items: for X, Y in MyDictionary.items(): + +=========================================== + +Change import path from command line +export PYTHONPATH=~/gene/wqflask (or whatever) + +=========================================== + +Create symbolic link: +ln -s dir_to_link_to linking_dir + +=========================================== + +Reboot server +sudo reboot + +=========================================== + +Using grep: + +grep -ir (search string) (directory) + +-i ignores case +-r searches recursively through subdirectories of directory you specify + +=========================================== + +Command line arguments: + +Use argparse to deal with command line arguments (instead of argv or optparse) + +=========================================== + +Change owner/group: + +chown zas1024 somefile (change owner of somefile to zas1024) +chown zas1024: somefile (change owner and group of somefile to zas1024) +sudo chown root: somefile (change owner/group of somefile to root) + +Change permissions: + +chmod g-wr (remove write/read privileges for group) +chmod g+w (add write privileges for group) +chmod u+rwx (add read, write, and execute privileges for user) +etc. + +=========================================== + +MySQL Things - + +Create fulltext (MATCH AGAINST syntax) index: +CREATE FULLTEXT INDEX __________ ON Table (Column); + +If <4 character searches not working: +Add ft_min_word_len = 2 (to make te minimum characters 2) to config file +Rebuild relevant tables with fulltext indexes with "REPAIR TABLE _______ QUICK" + +=========================================== + +To delete all .pyc files (for example if there's some issue you're having trouble resolving): +find . -name '*.pyc' -delete +http://stackoverflow.com/questions/785519/how-do-i-remove-all-pyc-files-from-a-project + diff --git a/doc/notes_DA.txt b/doc/notes_DA.txt new file mode 100644 index 00000000..410e0182 --- /dev/null +++ b/doc/notes_DA.txt @@ -0,0 +1,10 @@ +Danny's notes about the genenetwork source + +Location of static files: + +Location of HTML templates: wqflask/wqflask/templates/ + +Entry point of the wqflask app: wqflask/wqflask/__init__.py + +Application routes: wqflask/wqflask/views.py + diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 00000000..39ee5652 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,36 @@ +BeautifulSoup==3.2.1 +Flask==0.9 +Flask-Login==0.1.3 +Flask-Mail==0.7.6 +Flask-Principal==0.3.4 +Flask-SQLAlchemy==0.16 +Flask-Security==1.6.0 +Flask-WTF==0.8.3 +Jinja2==2.6 +MySQL-python==1.2.4 +PyYAML==3.10 +#Reaper==1.0 +Reindent==0.1.1 +SQLAlchemy==0.8.0 +WTForms==1.0.3 +Werkzeug==0.8.3 +apache-libcloud==0.12.3 +argparse==1.2.1 +blinker==1.2 +cairosvg==1.0.15 +itsdangerous==0.17 +logging-tree==1.2 +logilab-astng==0.24.3 +logilab-common==0.59.1 +#numarray==1.5.2 +numpy==1.7.0 +passlib==1.6.1 +pp==1.6.3 +pylint==0.27.0 +redis==2.7.2 +requests==1.1.0 +scipy==0.11.0 +simplejson==3.0.7 +wsgiref==0.1.2 +yolk==0.4.3 +XlsxWriter==0.7.2 diff --git a/doc/todo.txt b/doc/todo.txt new file mode 100644 index 00000000..1d781b13 --- /dev/null +++ b/doc/todo.txt @@ -0,0 +1,2 @@ +- Ask Rob about potentially recoding qtlreaper +- Ask Rob about Probe/cellid traits \ No newline at end of file diff --git a/misc/gn_installation_notes.txt b/misc/gn_installation_notes.txt deleted file mode 100644 index 584080f7..00000000 --- a/misc/gn_installation_notes.txt +++ /dev/null @@ -1,346 +0,0 @@ -UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration - -Enable UFW: -sudo ufw enable -Check status of UFW: -sudo ufw status verbose - -Limit ssh: -sudo ufw limit ssh - -Enable http, https: -sudo ufw allow http -sudo ufw allow https - -=========================================== - -http://evilrouters.net/2011/02/18/using-etckeeper-with-git-on-ubuntu/ - -Install etckeeper: -sudo apt-get install etckeeper - -Edit /etc/etckeeper/etckeeper.conf to comment out VCS="bzr" and comment in VCS="git" - -Initialize repository: -sudo etckeeper init - -Do initial commit: -sudo etckeeper commit "Initial commit." - -=========================================== - -Get git: - -sudo apt-get install git - -=========================================== - -Clone code from git repository: -git clone http://github.com/zsloan/genenetwork.git gene - -Pull from branch in git repository: -git pull origin flask(or whatever the branch is) - -**"apt-cache search" searches for text in the description for various packages, while -"apt-file search" searches for files within the packages** - -Search for package with a specified file that can be installed with apt-get -apt-file search _______ - -============================================ - -Create trash directory: -mkdir ~/.trash - -============================================ - -Using dpkg - -Check which system packages are already installed: -dpkg -l | less - -=========================================== - -Installing virtualenv: -sudo pip install virtualenv - -Tell virtualenv which directory to use: -virtualenv ve27 - -Start up virtual environment: -source ~/ve27/bin/activate - -=========================================== - -Install libmysqlclient-dev (Mysql-Python dependency) -sudo apt-get install libmysqlclient-dev - -Install python-dev (numpy dependency) -sudo apt-get install python-dev - -Install scipy dependencies: -sudo apt-get install libatlas-base-dev gfortran g++ - -=========================================== - -Install pip: -sudo apt-get install python-pip - -REMEMBER TO SOURCE VE BEFORE INSTALLING - -Comment out in requirements.txt: -Reaper -numarray - -Before installing from requirements.txt, install numpy separately: -pip install numpy==1.7.0 (or whatever version we're using) - -Install from requirements.txt (after activating virtualenv): -pip install -r gene/misc/requirements.txt - -=========================================== - -Installing QTL Reaper - -wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb -mv -v qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb qtlreaper-1.1.1.tar.gz -tar xvf qtlreaper-1.1.1.tar.gz (to unzip) -mkdir /home/zas1024/ve27/include/python2.7/Reaper -python setup.py install - -=========================================== - -Installing numarray 1.5.2 - -wget http://downloads.sourceforge.net/project/numpy/Old%20Numarray/1.5.2/numarray-1.5.2.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%2520Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload&ts=1358978306&use_mirror=superb-dca2 -mv -v numarray-1.5.2.tar.gz?r=http:%2F%2Fsourceforge.net%2Fprojects%2Fnumpy%2Ffiles%2FOld%20Numarray%2F1.5.2%2Fnumarray-1.5.2.tar.gz%2Fdownload numarray-1.5.2.tar.gz -tar xvf numarray-1.5.2.tar.gz -sudo python setup.py install - -=========================================== - -Installing nginx - -sudo -s -nginx=stable # use nginx=development for latest development version -echo "deb http://ppa.launchpad.net/nginx/$nginx/ubuntu lucid main" > /etc/apt/sources.list.d/nginx-$nginx-lucid.list -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys C300EE8C -apt-get update -apt-get install nginx - -Create configuration file in ~/gene/wqflask/other_config/nginx_conf/ (modeled off of the others) -Create symbolic link to config file in /etc/nginx/sites-enabled/: -ln -s dir_to_link_to linking_dir - -Run nginx: -sudo /usr/sbin/nginx - -============================================ - -Copy over zach_settings.py to /home/zas1024 directory (or whatever is home directory) - -export WQFLASK_SETTINGS=~/gene/wqflask/cfg/zach_settings.py (or wherever file is located) - -============================================ - -Install MySQL Server - -sudo apt-get install mysql-server - -mkdir /mnt/big -fdisk /dev/sdb -m: for help -n: new partion -... -w: write to table and exit - -Start MySQL server: -service mysql start - -Stop MySQL server: -service mysql stop - -Change root password: -mysql> UPDATE mysql.user SET Password=PASSWORD('your password') WHERE User='root'; - -Setup accounts in MySQL (first need to delete anonymous/non-root accounts): -#; use mysql; -#; select * from user; -#; delete from user where Host!="localhost"; -#; delete from user where User!="root"; -#; update user set Password = Password('yourpassword') where User='root'; -#; GRANT ALL ON *.* TO 'yourusername'@'%' IDENTIFIED BY 'yourpassword'; -#; select * from user; - -Use Percona to optimise my.cfg file: -https://tools.percona.com/wizard - -To check # CPUs: -grep -c processor /proc/cpuinfo - -To check memory, can use htop - -============================================ - -Check RSA key: -ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key - -03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) - -============================================ - -Installing redis server - -In the dls directory: -$ wget http://redis.googlecode.com/files/redis-2.6.14.tar.gz -$ tar xzf redis-2.6.14.tar.gz -$ cd redis-2.6.14 -$ make - -Run redis (from inside redis-2.6.14 directory): -src/redis-server - -For the rest of the installation instructions, refer to: -http://redis.io/topics/quickstart - -To continue installation (from the README): - -In order to install Redis binaries into /usr/local/bin just use: - - % make install - -Make install will just install binaries in your system, but will not configure -init scripts and configuration files in the appropriate place. This is not -needed if you want just to play a bit with Redis, but if you are installing -it the proper way for a production system, we have a script doing this -for Ubuntu and Debian systems: - - % cd utils - % ./install_server - -The script will ask you a few questions and will setup everything you need -to run Redis properly as a background daemon that will start again on -system reboots. - -In the redis config file (/etc/redis/6379.conf) uncomment the following line: -bind 127.0.0.1 - -This is so only the local computer has access - -============================================ - -Check RSA key: -ssh-keygen -l -f /etc/ssh/ssh_host_rsa_key - -03:2c:d7:01:01:f0:31:3a:c8:df:e4:98:62:2c:59:d2 root@penguin (RSA) - -============================================ - -Using Yolk - -Install Yolk: -pip install yolk - -Check packages installed in this virtual environment: -yolk -l - -Checks packages that have updates available: -yolk -U - -=========================================== - -Installing yaml - -Install libyaml-dev: -sudo apt-get install libyaml-dev - -Install yaml: -pip install pyyaml - -=========================================== - -Install MySQL Client - -To fix error "mysql_config not found" while installing packages with pip: -sudo apt-get install libmysqlclient-dev - -=========================================== - -Installing R - -sudo apt-get install r-base-dev - -=========================================== - -Installing rpy2 - -pip install rpy2 - -=========================================== - -Install Parallel Python (pp) - -wget http://www.parallelpython.com/downloads/pp/pp-1.6.3.tar.gz -tar xvf pp-1.6.3.tar.gz -python setup.py install - -========================================== - -Install requests - -pip install requests - -========================================== - -Install logging_tree - -pip install logging_tree - -========================================== - -Install colordiff (just to make it easier to view differences in files, not necessary for GN) - -sudo apt-get install colordiff - -========================================== - -Install NTP (network time protocol) - -sudo apt-get install ntp - -========================================== - -To get server running: - -!If having seemingly inexplicable problems with imports, make sure I've started the environment! - -Start up virtual environment: -source ~/ve27/bin/activate============== - -Install requests - -pip install requests - -======================= - -To set WQFLASK_SETTINGS environment variable: -export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) - -To change screen environment variable (if man not working or to get color, for example): -export TERM=screen - -To search for commands in history if necessary: -history | grep "(whatever is being searched for)" - -Run web server: -/usr/local/nginx/sbin/nginx - -Run server: -python runserver.py - -=========================================== - -To do full upgrade (as opposed to apt-get upgrade) -sudo aptitude full-upgrade - -=========================================== \ No newline at end of file diff --git a/misc/new_variable_names.txt b/misc/new_variable_names.txt deleted file mode 100644 index c11c160e..00000000 --- a/misc/new_variable_names.txt +++ /dev/null @@ -1,6 +0,0 @@ -RISet/riset -> group -webqtlDataset.py -> data_set.py -webqtlDataset (class object) -> DataSet -database/db -> dataset/data_set -DataEditingPage -> show_trait.py/show_trait.html -webqtlTrait -> GeneralTrait \ No newline at end of file diff --git a/misc/notes.txt b/misc/notes.txt deleted file mode 100644 index f8ce2759..00000000 --- a/misc/notes.txt +++ /dev/null @@ -1,311 +0,0 @@ -Clone code from git repository: -git clone http://github.com/zsloan/genenetwork.git gene - -Pull from branch in git repository: -git pull origin flask(or whatever the branch is) - -Add all files in current directory branch to be staged to be committed -git add -A - -Git push to my github repository -git push https://github.com/zsloan/genenetwork.git master - -Get the default path being used when a command is typed: -which _______ - -Change default path python looks at when importing -export PYTHONPATH="/home/zas1024/gene/wqflask" - -Get the paths checked when a user types a command: -echo $PATH - -Get all environment variables: -set - - -**** apt-get is an Ubuntu system-wide package manager, while pip is a Python-only system -for python packages **** - -Upgrade system packages for security updates, etc: -apt-get update -apt-get dist-upgrade (can probably be used in place of apt-get upgrade) -apt-get upgrade - -**"apt-cache search" searches for text in the description for various packages, while -"apt-file search" searches for files within the packages** - -Show information about given result item returned by apt-cache search: -apt-cache show _______ - -Search for package with a specified file that can be installed with apt-get -apt-file search _______ - -============================================ - -Check which system packages are installed: -dpkg -l | less - -============================================ - -NEVER USE SUDO WITH PIP - -To fix error "mysql_config not found": -sudo apt-get install libmysqlclient-dev - -Create requirements.txt file: -pip freeze > requirements.txt (or whatever filename) - -Install from requirements.txt: -pip install -r gene/wqflask/requirements.txt -t ve27 - -============================================ - -To get server running: - -!If having seemingly inexplicable problems with imports, make sure I've started the environment! - -Note: Virtual environment is not activated when running as root - -Start up virtual environment: -source ~/ve27/bin/activate - -Deactivate virtual environment: -deactivate (just type deactivate) - -To set WQFLASK_SETTINGS environment variable: -export WQFLASK_SETTINGS=~/zach_settings.py (or wherever file is located) - -zach_settings.py is the local config file with the DB_URI -default_settings.py is the config file with the configuration information that would apply -to any running instance of GeneNetwork - -To change screen environment variable (if man not working or to get color, for example): -export TERM=screen - -To search for commands in history if necessary: -history | grep "(whatever is being searched for)" - -Reload web server: -/usr/sbin/nginx -s reload - -Run server: -python runserver.py - -Run sendmail.py -python send_mail.py - -=========================================== - -UFW - default firewall confirguation tool for Ubuntu; eases iptables firewall configuration - -Enable UFW: -sudo ufw enable -Check status of UFW: -sudo ufw status verbose - -=========================================== - -Update user password: -use mysql; -UPDATE user SET password=PASSWORD('whatever') WHERE user='webqtl'; -FLUSH PRIVILEGES; - -=========================================== - -Redis things - -Restart redis server: -sudo /etc/init.d/redis_6379 stop -sudo /etc/init.d/redis_6379 start - -=========================================== - -Run pylint: -pylint file_name - -For options: -pylint --help - -=========================================== - -Start screen session: - -If "no option -R" or running for first time: -byobu-select-backend -2. screen - -byobu -RD (to start) -control-a then :multiuser on -control-a then :acladd sam - -More info on sharing byobu screen with other users: -http://wiki.networksecuritytoolkit.org/nstwiki/index.php/HowTo_Share_A_Terminal_Session_Using_Screen - -control-a c to create channel - -type: screen -list for sessions -screen -r zas1024/25679.byobu - -or if only one: - -screen -r zas1024/ - -=========================================== - -Start up log: -Go to /tmp and tail -f flask_gn_log - -=========================================== - -Coffeescript Stuff: - -coffee -c (filename) -coffee -c -w . (to watch for changes and recompile in current directory; the "." is for current directory) -coffee --help (for information about setting options) - -=========================================== - -Unset ASKPASS when trying to git push - -unset SSH_ASKPASS - -=========================================== - -htop: Gives information on processes, cpu/memory load, etc -dstat: Also gives various system information, resource usage, etc -df: Reports file system disk space usage - -=========================================== - -ll h*: Finds items in the directory starting with h - -=========================================== - -du -hms * | sort -n : Gives size used by different directories --h: human readable --m: in megabytes (default) --s: summarize - -=========================================== - -rm -rfv - -=========================================== - -cp -a (archive; copies recursively and doesn't follow symbol links) - -i (interactive, prompts before overwrite) - -v (verbose) - -b (backup) - -mv (same as above, but with no -a) - -=========================================== - -Add user: -sudo adduser -Edit /etc/sudoers to give user root privileges - - -=========================================== - -tidyp - Improves/beautifies html code -tidyp -m -i -w 100 index_page.html - -=========================================== - -ps ax - View processes -ps ux (the "u" is for user) - -kill (process #) - -=========================================== - -Installing QTL Reaper - -wget http://downloads.sourceforge.net/project/qtlreaper/qtlreaper/1.1.1/qtlreaper-1.1.1.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fqtlreaper%2Ffiles%2Flatest%2Fdownload&ts=1358975786&use_mirror=iweb -tar xvf qtlreaper-1.1.1.tar.gz (to unzip) -python setup.py build -python setup.py install - -=========================================== - -Using Yolk - -Install Yolk: -pip install yolk - -Check packages installed in this virtual environment: -yolk -l - -Checks packages that have updates available: -yolk -U - -=========================================== - -Python stuff: - -Classes should always inherit "object" -To iterate through dictionary items: for X, Y in MyDictionary.items(): - -=========================================== - -Change import path from command line -export PYTHONPATH=~/gene/wqflask (or whatever) - -=========================================== - -Create symbolic link: -ln -s dir_to_link_to linking_dir - -=========================================== - -Reboot server -sudo reboot - -=========================================== - -Using grep: - -grep -ir (search string) (directory) - --i ignores case --r searches recursively through subdirectories of directory you specify - -=========================================== - -Command line arguments: - -Use argparse to deal with command line arguments (instead of argv or optparse) - -=========================================== - -Change owner/group: - -chown zas1024 somefile (change owner of somefile to zas1024) -chown zas1024: somefile (change owner and group of somefile to zas1024) -sudo chown root: somefile (change owner/group of somefile to root) - -Change permissions: - -chmod g-wr (remove write/read privileges for group) -chmod g+w (add write privileges for group) -chmod u+rwx (add read, write, and execute privileges for user) -etc. - -=========================================== - -MySQL Things - - -Create fulltext (MATCH AGAINST syntax) index: -CREATE FULLTEXT INDEX __________ ON Table (Column); - -If <4 character searches not working: -Add ft_min_word_len = 2 (to make te minimum characters 2) to config file -Rebuild relevant tables with fulltext indexes with "REPAIR TABLE _______ QUICK" - -=========================================== - -To delete all .pyc files (for example if there's some issue you're having trouble resolving): -find . -name '*.pyc' -delete -http://stackoverflow.com/questions/785519/how-do-i-remove-all-pyc-files-from-a-project - diff --git a/misc/notes_DA.txt b/misc/notes_DA.txt deleted file mode 100644 index 410e0182..00000000 --- a/misc/notes_DA.txt +++ /dev/null @@ -1,10 +0,0 @@ -Danny's notes about the genenetwork source - -Location of static files: - -Location of HTML templates: wqflask/wqflask/templates/ - -Entry point of the wqflask app: wqflask/wqflask/__init__.py - -Application routes: wqflask/wqflask/views.py - diff --git a/misc/requirements.txt b/misc/requirements.txt deleted file mode 100644 index 39ee5652..00000000 --- a/misc/requirements.txt +++ /dev/null @@ -1,36 +0,0 @@ -BeautifulSoup==3.2.1 -Flask==0.9 -Flask-Login==0.1.3 -Flask-Mail==0.7.6 -Flask-Principal==0.3.4 -Flask-SQLAlchemy==0.16 -Flask-Security==1.6.0 -Flask-WTF==0.8.3 -Jinja2==2.6 -MySQL-python==1.2.4 -PyYAML==3.10 -#Reaper==1.0 -Reindent==0.1.1 -SQLAlchemy==0.8.0 -WTForms==1.0.3 -Werkzeug==0.8.3 -apache-libcloud==0.12.3 -argparse==1.2.1 -blinker==1.2 -cairosvg==1.0.15 -itsdangerous==0.17 -logging-tree==1.2 -logilab-astng==0.24.3 -logilab-common==0.59.1 -#numarray==1.5.2 -numpy==1.7.0 -passlib==1.6.1 -pp==1.6.3 -pylint==0.27.0 -redis==2.7.2 -requests==1.1.0 -scipy==0.11.0 -simplejson==3.0.7 -wsgiref==0.1.2 -yolk==0.4.3 -XlsxWriter==0.7.2 diff --git a/misc/todo.txt b/misc/todo.txt deleted file mode 100644 index 1d781b13..00000000 --- a/misc/todo.txt +++ /dev/null @@ -1,2 +0,0 @@ -- Ask Rob about potentially recoding qtlreaper -- Ask Rob about Probe/cellid traits \ No newline at end of file -- cgit v1.2.3