Category Archives: Scripting

ftp/sftp – vsftpd

# VSFTPD chroot configuration

>> Create a no-shell user
useradd -d $HOME_PATH -s /sbin/nologin $FTPUSER && passwd $FTPUSER

!!!MAKE SURE TO CHMOD 755 the parent directory!!!

yum -y install vsftpd

chkconfig vsftpd on

sed -i -e 's/IPTABLES_MODULES=""/IPTABLES_MODULES="ip_conntrack_ftp"/g' /etc/sysconfig/iptables-config

modprobe ip_conntrack_ftp

echo "rack" >> /etc/vsftpd/vsftpd.chroot_list

mv /etc/vsftpd/vsftpd.conf /etc/vsftpd/vsftpd.conf.ORIG

cat >/etc/vsftpd/vsftpd.conf <<EOF
# vsftpd.conf - PASSIVE
anonymous_enable=NO
local_enable=YES
write_enable=YES
local_umask=022
dirmessage_enable=YES
xferlog_enable=YES
listen_port=21
connect_from_port_20=YES
xferlog_std_format=YES
listen=YES
pam_service_name=vsftpd
userlist_enable=YES
tcp_wrappers=YES
pasv_min_port=60000
pasv_max_port=65000

# Add in /etc/vsftpd/vsftpd.chroot_list who you do *NOT* want to be chrooted
chroot_local_user=YES
chroot_list_enable=YES
chroot_list_file=/etc/vsftpd/vsftpd.chroot_list

# RackConnect
# pasv_enable=YES
# pasv_min_port=60000
# pasv_max_port=60100
# pasv_address=<publicRCip> (might not be required)

# Logging
xferlog_enable=YES
log_ftp_protocol=NO
syslog_enable=NO
vsftpd_log_file=/var/log/vsftpd.log
EOF

>> Make sure  to comment out "auth   required    pam_shells.so" in /etc/pam.d/vsftpd (errors in authenticate users with /bin/false shell):
sed -i 's/^\(auth.*required.*pam_shells\.so.*$\)/#\1/' /etc/pam.d/vsftpd 

>> Enable firewall ports (in Rackconnect, open the same on the physical firewall):

iptables -I INPUT -p tcp --dport 21 -m comment --comment "FTP" -j ACCEPT
iptables -I INPUT -p tcp -m multiport --dports 60000:65000 -m comment --comment "FTP passive mode ports" -j ACCEPT
/etc/init.d/iptables save

>> Restart the service
service vsfptd restart


If -> vsftpd: refusing to run with writable root inside chroot ()
=> allow_writable_chroot=YES

=======================================================

SFTP Jailed: 
!!!! remember that the users home directory must be owned by root 

groupadd sftponly

>> 1 domain managed by 1 or more users:
    useradd -d /var/www/vhosts -s /bin/false -G sftponly bob

>> 1 user managing multiple domains:
    useradd -d /var/www/vhosts -s /bin/false -G sftponly bob

SFTPUSER=bob
SFTPUSERPASS=$(tr -cd '[:alnum:]' < /dev/urandom | fold -w12 | head -n1)
echo "$SFTPUSERPASS" | passwd --stdin $SFTPUSER && echo -e "\nsfptuser: $SFTPUSER\npassword: $SFTPUSERPASS" 


>> /etc/ssh/sshd_config
#Subsystem sftp /usr/libexec/openssh/sftp-server
Subsystem sftp internal-sftp

>> 1 domain managed by 1 or more users:
Match Group sftponly
   ChrootDirectory %h
   X11Forwarding no
   AllowTCPForwarding no
   ForceCommand internal-sftp

>> 1 user managing multiple domains:
    Match Group sftponly
         ChrootDirectory /var/www/vhosts/%u
         X11Forwarding no
         AllowTCPForwarding no
         ForceCommand internal-sftp

sshd -t
service sshd restart 

>> Set correct permissions!!!
chmod 755 /var/www/
chown root:root /var/www
chown -R root:sftponly /var/www/*
find /var/www/ -type d | xargs chmod 2775
find /var/www/ -type f | xargs chmod 644

 

One liners to automatic creation of username and passwords

Automatic creation of users/passwords (FTP)

Manually create list.txt with user:doc_root
e.g.:

mydomain.com:/var/www/vhost/mydomain.com
example.com:/var/www/vhost/example.com

Get commands to create FTP users

cat list.txt | awk -F: '{print "useradd -d ",$2, "-s /bin/false -c TICKET_NUMBER ",$1 }'

 

Get commands to set FTP permissions (if doc_root exists already)

cat list.txt | awk -F: '{print "chown -R",$1, $2 }'

 

Generate and Assign random passwords to the users.

# for USER in $(awk -F: '{print $1}' list.txt) ; do PASS=$(tr -cd '[:alnum:]' < /dev/urandom | fold -w12 | head -n1) ; echo $PASS | passwd --stdin $USER ; echo -e "username: $USER\npassword: $PASS\n" | tee -a pass.txt ; done ; echo -e "\n========================\nHere the credentials:" ; cat pass.txt

 


Create a list of vhosts’ paths: vhosts.txt

Example with only .com domains:
/var/www/domain1.com
/var/www/domain2.com
/var/www/domain3.com

Use a regex for sed to extract the vhost name, removing dots (example based on the example above)
This will return a list of PATH and VHOSTNAME. We will use VHOSTNAME as USER for that path

for i in `cat vhosts.txt` ; do echo "$i" | tr '\n' ' ' ; echo "$i" | sed 's/^.*www\/\(.*\)com.*$/\1/' | sed 's/\.//g' ; done >> list.txt

 

Print out the commands to run to add FTP users (no SSH)
Once checked the output, run these lines

cat list.txt | awk '{print "useradd -d ",$1, "-s /bin/false -c COMMENT_HERE ",$2 }'

(for sftp only):

cat list.txt | awk '{print "useradd -d ",$1, "-s /bin/false -G sftponly -cCOMMENT_HERE ",$2 }'

 

This will print out commands to run to assign user:apache_group to the vhosts’ paths

cat list.txt | awk '{print "chown -R ",$2 ":www-data ",$1 }'

(for sftp only):

cat list.txt | awk '{print "chown root:root",$1 }'
cat list.txt | awk '{print "chown -R ",$2":"$2 ,$1"/*"}'

 

Set g+s on vhosts to preserve directory owner
[TO CHECK]

for i in `cat list.txt` ; do echo "chmod g+s $i" ; done

[THIS EXECUTE]

for i in `cat list.txt` ; do chmod g+s "$i" ; done

 

Create list of random passwords using pwgen

for i in `cat list.txt` ; do p=$(pwgen -s -B 16 1) ; echo "$i:$p" ; done > list_u_p.txt

 

Create list of random passwords using openssl

for i in `cat list.txt` ; do p=$(openssl rand -base64 12) ; echo "$i:$p" ; done > list_u_p.txt

 

Apply these passwords automatically

for i in `cat list_u_p.txt` ; do USER=`echo "$i" | awk -F":" '{print $1}'` ; PASS=`echo "$i" | awk -F":" '{print $2}'` ; echo -e "$PASS\n$PASS" | passwd "$USER" ; done

 

Print output for reference

hostname ; cat list_u_p.txt | awk -F":" '{print "\nusername:", $1, "\npassword:", $2}'

Space utilised one liners

# Current  folder space
du -sh <path>

# 10 biggest folders
du -m <path> | sort -nr | head -n 10

# Check high directories usage.
du -hcx --max-depth=5 | grep [0-9]G | sort -nr

# Exclude a path from the final calculation
cd /path
du -sh --exclude=./relative/path/to/uploads

# Check APPARENT size
du -h --apparent-size /path/file



# Check how much space is "wasted":
lsof | grep deleted | sed 's/^.* \(REG.*deleted.*$\)/\1/' | awk '{print $5, $3}' | sort | uniq | awk '{sum += $2 } END { print sum }'

# >> *if* the number is like "1.5e+10", you might need to use this to see that converted in MB or GB
lsof | grep deleted | sed 's/^.* \(REG.*deleted.*$\)/\1/' | awk '{print $5, $3}' | sort | uniq | awk '{sum += $2 } END { print sum " bytes - " sum/1024**2 " MB - " sum/1024**3 " G" }'

# Check the biggest files:
lsof | grep deleted | sed 's/^.* \(REG.*deleted.*$\)/\1/' | awk '{print $5, $3}' | sort | uniq | awk '{print $2, $1}' | sort -nr

>> than you can grep the file name from the output of "lsof | grep deleted" and check for the PID that holds that file (second column)
>> and issue the following command:
kill -HUP <PID>
>> And check again. This should release the used file.

 

Apparent size is the number of bytes your applications think are in the file. It’s the amount of data that would be transferred over the network (not counting protocol headers) if you decided to send the file over FTP or HTTP. It’s also the result of cat theFile | wc -c, and the amount of address space that the file would take up if you loaded the whole thing using mmap.

Disk usage is the amount of space that can’t be used for something else because your file is occupying that space.

In most cases, the apparent size is smaller than the disk usage because the disk usage counts the full size of the last (partial) block of the file, and apparent size only counts the data that’s in that last block. However, apparent size is larger when you have a sparse file (sparse files are created when you seek somewhere past the end of the file, and then write something there — the OS doesn’t bother to create lots of blocks filled with zeros — it only creates a block for the part of the file you decided to write to).

Source (clarification): http://stackoverflow.com/questions/5694741/why-is-the-output-of-du-often-so-different-from-du-b 

Holland backup setup

>> package:

yum install holland-mysqldump

>> Auto install script:

#!/bin/bash
which yum
rc=$?
if [ $rc != 0 ] ; then
    apt-get install -y python-setuptools python-mysqldb
else
    yum install -y MySQL-python
fi

if [ ! -f holland-1.0.10.tar.gz ] ; then
    wget http://hollandbackup.org/releases/stable/1.0/holland-1.0.10.tar.gz
fi
if [ ! -d holland-1.0.10 ] ; then
    tar zxf holland-1.0.10.tar.gz 
fi

cd holland-1.0.10
python setup.py install 

cd plugins/holland.lib.common/
python setup.py install

cd ../holland.lib.mysql/
python setup.py install

cd ../holland.backup.mysqldump/
python setup.py install

cd ../../config
mkdir -p /etc/holland/providers
mkdir -p /etc/holland/backupsets
cp holland.conf  /etc/holland/
cp providers/mysqldump.conf /etc/holland/providers/
cp backupsets/examples/mysqldump.conf /etc/holland/backupsets/default.conf

cd /etc/holland
sed -i 's=/var/spool/holland=/var/lib/mysqlbackup=g' holland.conf
sed -i 's/backups-to-keep = 1/backups-to-keep = 7/g' backupsets/default.conf
sed -i 's/file-per-database.*=.*no/file-per-database = yes/g' backupsets/default.conf
sed -i 's/file-per-database.*=.*no/file-per-database = yes/g' providers/mysqldump.conf

hour=$(python -c "import random; print random.randint(0, 4)")
minute=$(python -c "import random; print random.randint(0, 59)")
echo '# Dump mysql DB to files. Time was chosen randomly' > /etc/cron.d/holland
echo "$minute $hour * * * root /usr/local/bin/holland -q bk" >> /etc/cron.d/holland

mkdir -p /var/lib/mysqlbackup
mkdir -p /var/log/holland
chmod o-rwx /var/log/holland

echo 
echo
echo "Holland should be installed now."
echo "Test backup: holland bk"
echo "See cron job: cat /etc/cron.d/holland"
echo "See backups: find /var/lib/mysqlbackup/"

>> Where is the DB conf file:

/etc/holland/backupsets

>> When it runs

cat /etc/cron.d/holland

>> Test command to see if all works without actually run it

holland bk --dry-run

>> /etc/holland/backupsets/default.conf

[mysql:client]
defaults-extra-file = /root/.my.cnf

 

Rackspace Cloud – Remove old System IDs via command line

Rough script/instructions 🙂

>> set your variables:
TOKEN=""
REGION="lon"
DDI=""  < this is the account number

>> Generate a list of backup agents
curl -sH  "X-Auth-Token: $TOKEN" -H "Content-type: application/json" -X GET https://$REGION.backup.api.rackspacecloud.com/v1.0/$DDI/user/agents | python -m json.tool | egrep "MachineName|MachineAgentId" | awk -F":" '{print $2}' | sed 's/ //g' | sed '{N;s/\n//}' > list.txt

>> Manually remove WANTED backup agents (leave only the ones you want to remove):
vim list.txt 

>> Generate remove list
awk -F, '{print $1}' list.txt > remove.txt


>> generate the exec file to review
for AGENTID in `cat remove.txt`; do echo curl -sH \"X-Auth-Token: $TOKEN\" -H \"Content-type: application/json\" -X POST https://$REGION.backup.api.rackspacecloud.com/v1.0/$DDI/agent/delete -d \'{\"MachineAgentId\": $AGENTID}\' ; done >> exec_me

>> exec the API calls
/bin/bash exec_me

 

MySQL Engine – InnoDB vs MyISAM

InnoDB does locking on the row level and runs queries as nonlocking consistent reads by default

How to check the engine of tables for a database

mysql> select engine from tables.information_schema where table_schema='<DB_NAME>';

 


Summary of Engines’ space utilised

select engine,sum(index_length+data_length)/1024/1024,count(engine) from information_schema.tables group by engine;

 

Check MyISAM tables

NOTE:

When it converts from MyISAM to InnoDB, you will see in process list ‘copy to tmp table’. This has nothing to do with tmp_table_size or tmpdir. What it does, it goes inside your /var/lib/mysql/<dbname>/ and creates temporary files called like #sqlXXXX.frm and #sqlXXXX.ibd. Keep an eye on the size of the .ibd file to monitoring the progression. It should be roughly big like the sum of <table>.MYI and <table>.MYD. +~20%

 

SELECT table_schema,table_name FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA NOT IN ("information_schema", "performance_schema", "mysql") and engine = 'MyISAM';

Change all to InnoDB
https://support.rackspace.com/how-to/mysql-engines-myisam-vs-innodb/
When?
https://docs.dev4press.com/tutorial/wordpress/choosing-mysql-database-engine-myisam-or-innodb/
Quick answer: Always EXCEPT If you want to use FULLTEXT or SPATIAL indexes for some tables: in this case there is no choice and you must use MyISAM.

NOTE: FULLTEXT index support for InnoDB tables requires MySQL 5.6.4 or higherMariaDB 10.0.5 or higher
Highly recommend upgrading to MariaDB 10.0 as this is the 5.6 MySQL equivalent. It does not alter the data structures as 5.6 does and can be downgraded much more easily. It contains all the features of 5.6 plus more as well as being a drop-in replacement for MySQL in RHEL7.
I would point you to these KB articles:
https://mariadb.com/kb/en/mariadb-vs-mysql-features/https://mariadb.com/kb/en/mariadb/mariadb-vs-mysql-compatibility/Repositories: https://downloads.mariadb.org/mariadb/repositories/#mirror=somerset

Check if there are FULLTEXT or SPATIAL indexes:

mysql> use information_schema;
mysql> SELECT DISTINCT CONCAT(STATISTICS.TABLE_SCHEMA, '.', STATISTICS.TABLE_NAME) AS fulltext_table, (TABLES.DATA_LENGTH + TABLES.INDEX_LENGTH)/1024/1024 AS size_M FROM STATISTICS INNER JOIN TABLES ON TABLES.TABLE_SCHEMA = STATISTICS.TABLE_SCHEMA AND TABLES.TABLE_NAME = STATISTICS.TABLE_NAME WHERE STATISTICS.INDEX_TYPE IN ("SPATIAL", "FULLTEXT") GROUP BY STATISTICS.TABLE_SCHEMA, STATISTICS.TABLE_NAME;

Convert ALL MyISAM tables to InnoDB (all databases):

2 steps + backup

# mysql --batch --skip-column-names --execute 'select concat("alter table ",TABLE_SCHEMA,".",TABLE_NAME," ENGINE='MyISAM';") from information_schema.TABLES where ENGINE = "MyISAM" AND TABLE_SCHEMA NOT IN ("information_schema", "performance_schema", "mysql");' > rollback_all2Innodb.sql
# mysql --batch --skip-column-names --execute 'select concat("alter table ",TABLE_SCHEMA,".",TABLE_NAME," ENGINE='InnoDB';") from information_schema.TABLES where ENGINE = "MyISAM" AND TABLE_SCHEMA NOT IN ("information_schema", "performance_schema", "mysql");' > all2Innodb.sql
# mysql < all2Innodb.sql

(one liner – CAREFUL!)

# mysql --batch --skip-column-names --execute 'select concat("alter table ",TABLE_SCHEMA,".",TABLE_NAME," ENGINE='InnoDB';") from information_schema.TABLES where ENGINE = "MyISAM" AND TABLE_SCHEMA NOT IN ("information_schema", "performance_schema", "mysql");' | mysql

 

Convert MyISAM tables of a SPECIFIC database to InnoDB:

Backup and convert-file

# DB="<dbname>"
# mysql --batch --skip-column-names --execute "select concat('alter table ',TABLE_SCHEMA,'.',TABLE_NAME,' ENGINE = \'MyISAM\';') from information_schema.TABLES where TABLE_SCHEMA = '"$DB"' and ENGINE = 'MyISAM';" > rollback_$DB_all2Innodb.sql

# mysql --batch --skip-column-names --execute "select concat('alter table ',TABLE_SCHEMA,'.',TABLE_NAME,' ENGINE = \'InnoDB\';') from information_schema.TABLES where TABLE_SCHEMA = '"$DB"' and ENGINE = 'MyISAM';" > $DB_all2Innodb.sql

NOTE: When it converts from MyISAM to InnoDB, you will see in process list ‘copy to tmp table’. This has nothing to do with tmp_table_size or tmpdir. What it does, it goes inside your /var/lib/mysql/<dbname>/ and creates temporary files called like #sqlXXXX.frm and #sqlXXXX.ibd. Keep an eye on the size of the .ibd file to monitoring the progression. It should be roughly big like the sum of <table>.MYI and <table>.MYD. +~20%

Find files based on date/time

# ONE LINERS


> Modified in the last 12 hours (720 min)
find . -cmin -720 

> Modified in the last day
find . -mtime -1



# => ctime - for hacked/modified files 
# look for ctime instead, hacked scripts can't set that to what they want as opposed to mtime:

find -cmin -$n_minutes_ago
find -ctime -$n_days_ago
ls -lc   ## sorted by name
ls -ltc   ## sorted by time


>> File OLDER THAN xx days:
find . -type f -ctime +$n_days_ago

>> Find files RESTORED older that xx days and MOVE them
find . -type f -mtime +$n_days_ago | xargs -I '{}' mv {} /destination/path/