Place Ghantoos

Here is a list of commands/tips I collected over the past couple of month. I will be updating it every once in while.

Please don’t hesitate to comment suggesting/correcting commands.

I hope this can be useful!

Cheers,
Ignace M -ghantoos-

# SHELL: recursive md5sum of files in directories

find * -type f -exec md5sum {} \;

# PYTHON: copy list content

list_copy = original[:]

# SHELL: ssh completion with /etc/hosts

put in /etc/bash.bashrc or your .bashrc

complete -W "$(cat /etc/hosts | awk '$1 != "#" {print $2}')" ssh

# SHELL: recursive copy via tar:

(cd /répertoire_source && tar cfp - . ) | (cd /répertoire_cible && tar xvpf -)

# FREEBSD: update ports tree and ports

portsnap fetch update
## Build a package when each specified port is installed or upgraded
LC_ALL=C portupgrade -ap

## Use packages instead of ports whenever available
LC_ALL=C portupgrade -aP

## use -f to force the upgrade of a package even if it is to be a downgrade or just a reinstall of the same version

# FREEBSD: list packages to be upgraded

# after portsnap fetch update
pkg_version -vIL=

# SHELL: get partition info (e.g. last fsck date)

sudo dumpe2fs -h /dev/hda3

# SHELL: convert mp4/avi to flv(ffmpeg)

#!/bin/sh

for file in `ls *.mp4`;do ffmpeg -i $file -y -b 800 -r 40 -f flv -vcodec flv -ab 128 -ar 44100 $file.flv;done;
rename -s/.mp4.flv/.flv/ *.mp4.flv

# SHELL: convert flv to avi/mpeg

# AVI (on FreeBSD)
for file in `ls *.flv`;do ffmpeg -i $file -ab 56 -ar 22050 -b 500 -s 320x240 -aspect 4:3 -vcodec libxvid -acodec libmp3lame $file.avi;done;
rename -s/.flv.avi/.avi/ *.flv.avi

# MPEG (on FreeBSD)
for file in `ls *.flv`;do ffmpeg -i $file -ab 56 -ar 22050 -b 500 -s 320x240 $file.mpg
rename -s/.flv.mpg/.mpg/ *.flv.mpg

# ON DEBIAN/UBUNTU
for file in `ls *.flv`;do mencoder $file -ovc xvid -xvidencopts bitrate=1000:autoaspect -vf pp=lb -oac mp3lame -lameopts fast:preset=standard -o $file.avi;done;
rename 's/.flv.avi$/.avi/' *.flv.avi

# TO MP4
for file in `ls *.mp4`;do ffmpeg -i $file -y -vcodec libx264 -s 640x480 -maxrate 1800k -b 800k $file.mp4;done;

# SHELL: convert flv to mp3 (ffmpeg)

ffmpeg -i flashvideo.flv -f mp3 -vn -acodec copy -ar 44100 -ab 160 -ac 2 output.mp3  

for file in `ls *.flv`;do ffmpeg -i $file -f mp3 -vn -acodec copy -ar 44100 -ab 160 -ac 2 $file.mp3;done;
rename 's/.flv.mp3$/.mp3/' *.flv.mp3

# SHELL: backup/restore over SSH using tar

cd /
tar cvpzf - --exclude=proc --exclude=lost+found --exclude=mnt --exclude=sys --exclude=mystuff * | ssh ghantoos@home.ghantoos.org "cat > /home/ghantoos/root.tar.gz"
---------------------------
tar xvpfz root.tar.gz -C /

# without compression
tar cvf - /toto/titi | ssh localhost "cd tmp && tar xvf -"

# restore (on host to be restored)
cd /
ssh root@192.168.1.201 "cat /home/backup/bla.tar.gz" | tar zxvf - 

# FREEBSD: mount a freebsd UFS partition under a Linux

mount -r -t ufs -o ufstype=ufs2 /dev/sda1 /tmp/bla/

# SHELL: diff the output of commands

diff <(command1) <(command2)

# SHELL: get partition journal size (ext2/3)

# step 1
sudo tune2fs -l /dev/sdb5 | grep -i "journal inode"
Journal inode:            8

# step 2
sudo /sbin/debugfs -R 'stat <8>' /dev/sdb5 | grep -i size
debugfs 1.40.8 (13-Mar-2008)
User:     0   Group:     0   Size: 134217728
Fragment:  Address: 0    Number: 0    Size: 0

# SHELL: mp3 boost volume (increase)

mp3gain -r plop.mp3

# LINUX: check for rootkits

sudo rkhunter -c

sudo chkrootkit

# SHELL: convert flv to avi (mencoder)

mencoder bla.flv -ovc lavc -oac pcm -lavcopts vcodec=mpeg4 -o bla.avi

# SHELL: backup using rdiff-backup

# backup over SSH
sudo rdiff-backup -v5 --print-statistics --preserve-numerical-ids --exclude /proc --exclude /sys --exclude /mnt /  ghantoos@home.ghantoos.org::/media/WD_ext/rdiff-backups/heliopolis/

# Local backup
** my home (max 1 mega)
sudo rdiff-backup -v5 --print-statistics --preserve-numerical-ids --max-file-size 1048576 /home/ /media/WD_ext/rdiff-backups/home/
** my system
sudo rdiff-backup -v5 --print-statistics --preserve-numerical-ids --exclude /proc --exclude /sys --exclude /home --exclude /mnt / /media/WD_ext/rdiff-backups/zaatar/

# Restore your data
** to 3 days ago
rdiff-backup -v5 --print-statistics --force -r 3D /path/to/backup /mnt/target/
** to the last backup
rdiff-backup -v5 --print-statistics --force -r now /path/to/backup /mnt/target/

# SHELL: sort directories by size

du -sh * | sort -nr

#search only on current partition
du -x / | sort -n

# SHELL: mount squashfs image

mount myfiles.sqsh /mnt/mydir -t squashfs -o loop

# SHELL: format ext3 partition limiting superuser reserved blocks

 mkfs.ext3 -m 0.5 -L namus /dev/sdX1

# SHELL: change partition label

# EXT2/3
sudo e2label /dev/sdb1 my_external

# FAT32
sudo mlabel -i /dev/sdb1 ::my_external

# NTFS
sudo ntfslabel /dev/sdb1 my_external

# SHELL: remove specific files in all subdirectories (e.g. desktop.ini)

find /media/bla -name desktop.ini -exec rm -vf {} \;

# SHELL: rsync local/over SSH

# over SSH: don't delete files on destination
rsync -avz --exclude="lost+found" --stats --progress -e ssh /media/camera/ yourdomain.com:camera/

# over SSH: delete files on destination
rsync -avz --exclude="lost+found" --delete --stats --progress -e ssh /media/camera/ yourdomain.com:camera/

# local rsync: dry run (always useful..)
rsync -avz --exclude="lost+found" --dry-run --stats --progress --delete /media/camera/ /media/Bacchus/camera/

# local rsync (careful this deletes non-existing files on destination!)
rsync -avz --exclude="lost+found" --progress --delete /media/camera/ /media/Bacchus/camera/

# SHELL: nmap

# open ports
nmap -T 3 192.168.1.1

# check versions
nmap -sV 192.168.1.1

# OS
nmap -O 192.168.1.1

# SHELL: zip / unzip

# recursive on folder
zip -r folder.zip folder/

# extract
unzip folder.zip

# SHELL: capitalize file names

# rename file capitalized
rename 's/./u$&/g' *

# check without actually renaming
rename -n 's/./u$&/g' *

# capitalize using sed
ls | sed 's/./u&/g'

# on FreeBSD the synthax is
rename -s/./u$&/g *

# SHELL: dump/restore SQL database

# dump
mysqldump --add-drop-table -h localhost -u USERNAME -p DATABASENAME | bzip2 -c > ghantoos.org.sql.bz2

# restore
bzip2 -d ghantoos.org.sql.bz2
mysql -h localhost -u USERNAME -p DATABASENAME < ghantoos.org.sql

# SHELL: jump by word in shell

$ cat .inputrc
# use [ctrl-arrowright] or [ctrl-arrowleft] to jump between words
";5C": forward-word
";5D": backward-word

# SHELL: GNU screen “screen buffer” (for scrollback)

cat ~/.screenrc
termcapinfo xterm|xterms|xs|rxvt ti@:te@

# SHELL: CD ripping (audio)

# rip all CD
cdparanoia -B --

# rip track 5
cdparanoia -B 5

# SHELL: bash history

# add to ~/.bashrc
export HISTTIMEFORMAT="%Y.%m.%d (%H:%M) "

# which results in:
$ history
  1  2009.07.02 (16:25) history

# SHELL: test hard disk read speed

sudo hdparm -Tt /dev/sdc

# SHELL: CVS update comment

cvs admin -m 1.2:"New tests" Test/cvs.lines

# REDHAT: RPM tricks

# List files included in local rpm:
rpm -qvlp yum-utils-0.5-2.el4.centos.noarch.rpm

# List dependencies of local rpm:
rpm -qpR yum-utils-0.5-2.el4.centos.noarch.rpm 

# List post/pre install/uninstall script of a local rpm:
rpm -qp --scripts  yum-utils-0.5-2.el4.centos.noarch.rpm

# Find out to which rpm belongs a file :
rpm -qf /bin/bash
bash-3.0-19.3

# RPM queryformat
rpm -qa --queryformat '%-30{NAME}%-10{VERSION}\n'
rpm -qp --queryformat '%{PACKAGER}\n' bla-1.2.3.rpm
More tags here.

# Extract an rpm
rpm2cpio myrpmfile.rpm | cpio -idmv

# FREEBSD: list PCI hardware (lspci equivalent)

pciconf -lv

# FREEBSD: update using only packages (not ports)

sudo portupgrade -arvPP

# FREEBSD: fix ports DB inconsistencies

pkgdb -F

# DEBIAN: .deb tricks

# extract scripts from .deb:
ar x package.deb
tar xvfz control.tar.gz

# Find out to which deb belongs a file:
dpkg --search /usr/sbin/add-shell

# show dependencies or reverse dependencies of package
apt-cache depends gnome-terminal
apt-cache rdepends gnome-terminal

# show only installed reverse dependencies:
apt-cache rdepends gnome-terminal --installed

# List all installed unstable packages:
apt-show-versions | grep unstable

# List dependencies of all installed packages:
for item in `dpkg --get-selections | sed 's/\s.*//'`; do apt-cache depends $item >> /home/ghantoos/tmp/depends ; done

# Extract deb package:
ar vx mypackage.deb

# Print all depending packages with their versions:
for item in `apt-cache depends --installed lshell | grep "Depends:" | cut -d":" -f2`; do dpkg-query -W -f='${Package;-20}: ${Version}\n' $item;done;

# SHELL: gzip man pages

gzip -9 -c lshell.1 > lshell.1.gz

# PYTHON: Simple HTTP server

cd /your/dir/
python -m SimpleHTTPServer 8080

# BSD: pkgsrc

export CVSROOT=anoncvs@anoncvs.fr.netbsd.org:/cvsroot
export CVS_RSH=ssh
cd /usr
cvs -q co pkgsrc

#then
cd bootstrap/
./bootstrap

# SHELL: delete files older than XX days

find . -ctime -XX -name "*.ncap" -exec rm {} ;

# DEBIAN: backup/restore package list

# BACKUP:
dpkg --get-selections > /backup/installed-software.log

# RESTORE:
sudo apt-get update
sudo dpkg --clear-selections
dpkg --set-selections < /backup/installed-software.log
sudo dselect

# SHELL: reload .screenrc on the fly

# add the following to your .screenrc
bind R eval "source $HOME/.screenrc" "echo '.screenrc reloaded!'"

# from inside your screen, ctrl+a+R ('a' is my escape character) to reload your screenrc
# Thank you sbz for the tip!

# SHELL: create a patch using diff

# create a patch on a file
diff -uN file file.new > mypatch

# create a patch on a directory (recursively)
diff -ruN directory/ directory.new/ > mypatch

# add -p flag when patching C files

# SHELL: apply a patch

# apply the patch on a file
patch < mypatch

# apply a patch on a directory (recusively)
patch -R < mypatch

# if pathname in patch is not the same as yours, use -p to strip it e.g:
patch -p3 < mypatch

# PYTHON: split string with multiple separators

import re
plop='one;two three&four'
re.split('[;& ]',plop)
=> ['one', 'two', 'three', 'four']

# SHELL: irssi cool features

# hilight using a regexp
/hilight -regexp -actcolor %R myregexp

# show all hilights in a window
/lastlog -hilight

# ignore message matching a regexp
/ignore -regexp -pattern "myregexp" * MSGS

# ignore join,part,quit,etc. messages from channel
/ignore -channels #mychannel * JOINS PARTS QUITS NICKS

# SHELL: search and replace/delete in vi/vim

# search and replace
:%s/oldpattern/newpattern/

# search and delete
:g/pattern/d

# SHELL: convi statusbar in GNU screen

# add the following to your .screenrc
hardstatus alwayslastline "%-Lw%{= BW}%50>%n-%t%{-}%+Lw%< %=|%H|%d-%m %c"

# SHELL: execute application in loop refreshing the output

# watch a folder
watch ls

# watch established connections of your system
watch -n1 -d 'netstat -an | grep ESTABLISHED'

# SHELL: MySQL delete database

DROP DATABASE yourdbname;

# DEBIAN: install specific version using apt-get

apt-get install nautilus=2.2.4-1

# SHELL: prevent shell from expanding path (ignore shell expansion)

$ export GLOBIGNORE='*:?"
$ TOTO=/usr/bin/*
$ echo $TOTO
/usr/bin/*

# If GLOBIGNORE is set, each matching file name that also matches one of the patterns
# in GLOBIGNORE is removed from the list of matches.

# UNIX: add printer using cups web interface

http://localhost:631/

# UNIX: GnuPG tips

# list keys
gpg --list-keys

# edit key
gpg --edit-key  XXXXXXXX
Command> 

# delete UID
Command> uid 1
Command> deluid
Command> save

# add UID
Command> adduid
(...)
Command> save

# set UID as primary
Command> uid 1
Command> primary
(...)
Command> save

# generate a revocation certificate
gpg --out rev.asc --gen-revoke XXXXXXXX

# export key
gpg --armor --output signature.asc --export XXXXXXXX

# send key
gpg --keyserver wwwkeys.eu.pgp.net --send-key XXXXXXXX

# import key
gpg --keyserver wwwkeys.eu.pgp.net --recv-keys XXXXXXXX

# refresh all keys
gpg --refresh-keys

# check for signatures
gpg --edit XXXXXXXX
Command> check

# list a key's signatures
gpg --list-sig XXXXXXXX

# UNIX: execute as user (sudo)

sudo -u ghantoos ls -l

# UNIX: completely erase files (shred)

shred -f -v -z -u blabla

# on FreeBSD (part of sysutils/coreutils)
gshred -f -v -z -u

# find and shred all files in subdirectories
find /dir -type f -exec gshred -f -v -z -u '{}' \;;

# PYTHON: list all methods in class

>>> import bla
>>> dir(bla)

# SHELL: cryptsetup tips

# Add a passphrase (on a new slot)
sudo cryptsetup luksAddKey /dev/hda3

# Delete a passphrase from a slot
sudo cryptsetup luksDelKey /dev/hda3 0

# SHELL: tar.bz2 to tar.gz

bunzip2 foo.tar.gz
gzip -9 foo.tar

# DEBIAN: apt-*

apt-zip - Update a non-networked computer using apt and removable media
apt-show-version - Lists available package versions with distribution
apt-file - search for files within Debian packages

# SHELL: useful regex

# match line not containing word
((?!latest).*)

# DEBIAN: reportbug

reportbug --template --bts debian -S wishlist -s "New upstream version available bar-1.3.6" --email=foo@example.com bar

# DEBIAN: compare versions with dpkg

dpkg --compare-versions 2.2.2-1 lt 2.2.2-1.1; echo $?
# if it outputs 0 => true
# if it ouputs 1 => false

# DEBIAN: Use multiple CPU cores when building/compiling kernel

$ cat kernel-pkg.conf
# to use 4 cores:
CONCURRENCY_LEVEL := 4

# SHELL: Delete all empty directories

find -depth -type d -empty -exec rmdir {} \;

# LINUX: chroot mounts

mount /dev/md2 /mnt/
mount /dev/md1 /mnt/boot/
mount -o bind /dev/ /mnt/dev/ 
mount -o bind /dev/shm/ /mnt/dev/shm/
mount -o bind /proc/ /mnt/proc/   
mount -o bind /proc/ /mnt/proc 
mount -o bind /sys/ /mnt/sys/
chroot /mnt/

# DEBIAN: mp3 encoder

# from debian-mulimedia repository
sudo apt-get install gstreamer0.10-lame

# SHELL: sed, change first occurence in file

sed '0,/pattern/s//to_that/' file

# SHELL: convert ISO to UTF-8

iconv --from-code=ISO-8859-1 --to-code=UTF-8 ./oldfile.htm > ./newfile.html

# SHELL: Calculate size used by filetype / extension

find . -type f -regextype posix-egrep -iregex ".*(avi|mov|wmv)$" -exec wc -c {} +

# SHELL: Convert VirtualBox vdi to qcow2

qemu-img convert -O qcow2 deb-vpn-oecd.vdi deb-vpn-oecd.img

# SHELL: bash

## set terminal title (works with gnome-terminal)
## add in bashrc / profiles
export PROMPT_COMMAND='echo -ne "\033]0;"`hostname -s`"\007"'

# SHELL: rsync directories only (tree/structure)

rsync -avz --omit-dir-times --include='*/' --exclude='*' -e ssh /my/path/ rmyremotehost:/my/path/

# SHELL: drop all MySQL databases

mysql -uroot -p  -e "show databases" | grep -v Database | grep -v mysql | grep -v information_schema | gawk '{print "drop database " $1 ";select sleep(0.1);"}' | mysql 

# SHELL: lvcreate maximum size available

lvcreate -l 100%FREE -n lv_backup vg_backup

# SHELL: grep & tab

grep -E $'foo\tbar' myfile

# SHELL: What is my IP

curl -s http://whatismyip.org/

# DEBIAN: debug package installation errors

## .g. Setting up grub-pc (1.98+20100804-14+squeeze1) ...
## dpkg: error processing grub-pc (--install):
## subprocess installed post-installation script returned error exit status 1
DEBCONF_DEBUG=developer dpkg --configure --pending