diff --git a/bin/oldies/bin_incremental_backup.sh b/bin/oldies/bin_incremental_backup.sh new file mode 100755 index 0000000..3046bc3 --- /dev/null +++ b/bin/oldies/bin_incremental_backup.sh @@ -0,0 +1,25 @@ +#!/bin/bash +export TARGET=/opt/backups/X11952 +export SOURCE=/opt/bin +export LADATE=`date +%Y%m%d` +export HIER=`date +%Y%m%d -d "1 day ago"` + +export LEJOUR=`date +%d` +export TODELETE=`date +%Y%m%d -d "3 months ago"` + +SCRIPT=`basename $0 .sh` +FLAGFILE=$TARGET/flag_$SCRIPT.$LADATE + +if [ -f $FLAGFILE ] +then + echo $FLAGFILE found ... + exit 0 +fi + +HEURE=`date +%H:%M` +echo "--- $HEURE - BEGIN" >> $FLAGFILE + +/opt/bin/incremental_backup.sh + +HEURE=`date +%H:%M` +echo "--- $HEURE - END" >> $FLAGFILE diff --git a/bin/oldies/incremental_backup.sh b/bin/oldies/incremental_backup.sh new file mode 100755 index 0000000..0fc73da --- /dev/null +++ b/bin/oldies/incremental_backup.sh @@ -0,0 +1,49 @@ +#!/bin/bash +if [ ! -e $SOURCE ] +then + echo "$SOURCE n'existe pas !" + exit 1 +fi + +export LADATE=`date +%Y%m%d` +export HIER=`date +%Y%m%d -d "1 day ago"` + +export LEJOUR=`date +%d` +export TODELETE=`date +%Y%m%d -d "3 months ago"` + +DESTINATION=`basename $SOURCE` +if [ -d $TARGET/$LADATE/$DESTINATION ] +then + echo "ALREADY DONE : $TARGET/$LADATE" + exit 0 +fi + +if [ ! -d $TARGET/$HIER/$DESTINATION ] +then + # Le backup d'hier n'existe pas + # PC éteint, problème backup, etc. + # On prend le dernier backup comme référence + echo "NOT FOUND : $TARGET/$HIER/$DESTINATION" + LAST_ONE=`ls -tr $TARGET | tail -n 1` + echo $LAST_ONE + if [ ! -d $TARGET/$LAST_ONE/$DESTINATION ] + then + echo "SECOND CHANCE, NOT FOUND : $TARGET/$LAST_ONE/$DESTINATION" + exit 1 + fi + mv $TARGET/$LAST_ONE $TARGET/$HIER + ln -s $TARGET/$HIER $TARGET/$LAST_ONE +fi + +if [ -d $TARGET/$TODELETE/$DESTINATION ] +then + rm -rf $TARGET/$TODELETE/$DESTINATION +fi + +echo $TARGET/$LADATE/$DESTINATION + +mkdir -p $TARGET + +/usr/bin/rsync -a --no-o --delete --safe-links $EXCLUDED_DIRS --link-dest=$TARGET/$HIER/ $SOURCE $TARGET/$LADATE/ + +echo "DONE" diff --git a/bin/oldies/incremental_backup.sh.20180829 b/bin/oldies/incremental_backup.sh.20180829 new file mode 100755 index 0000000..0fc73da --- /dev/null +++ b/bin/oldies/incremental_backup.sh.20180829 @@ -0,0 +1,49 @@ +#!/bin/bash +if [ ! -e $SOURCE ] +then + echo "$SOURCE n'existe pas !" + exit 1 +fi + +export LADATE=`date +%Y%m%d` +export HIER=`date +%Y%m%d -d "1 day ago"` + +export LEJOUR=`date +%d` +export TODELETE=`date +%Y%m%d -d "3 months ago"` + +DESTINATION=`basename $SOURCE` +if [ -d $TARGET/$LADATE/$DESTINATION ] +then + echo "ALREADY DONE : $TARGET/$LADATE" + exit 0 +fi + +if [ ! -d $TARGET/$HIER/$DESTINATION ] +then + # Le backup d'hier n'existe pas + # PC éteint, problème backup, etc. + # On prend le dernier backup comme référence + echo "NOT FOUND : $TARGET/$HIER/$DESTINATION" + LAST_ONE=`ls -tr $TARGET | tail -n 1` + echo $LAST_ONE + if [ ! -d $TARGET/$LAST_ONE/$DESTINATION ] + then + echo "SECOND CHANCE, NOT FOUND : $TARGET/$LAST_ONE/$DESTINATION" + exit 1 + fi + mv $TARGET/$LAST_ONE $TARGET/$HIER + ln -s $TARGET/$HIER $TARGET/$LAST_ONE +fi + +if [ -d $TARGET/$TODELETE/$DESTINATION ] +then + rm -rf $TARGET/$TODELETE/$DESTINATION +fi + +echo $TARGET/$LADATE/$DESTINATION + +mkdir -p $TARGET + +/usr/bin/rsync -a --no-o --delete --safe-links $EXCLUDED_DIRS --link-dest=$TARGET/$HIER/ $SOURCE $TARGET/$LADATE/ + +echo "DONE" diff --git a/bin/oldies/quotidien.sh b/bin/oldies/quotidien.sh new file mode 100755 index 0000000..5446837 --- /dev/null +++ b/bin/oldies/quotidien.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# +# Les backups du serveur topisto.net +# +/opt/bin/recup_topisto_backups.sh + +# +# Les backups locaux +# +#/opt/bin/twitterbot_incremental_backup.sh +#/opt/bin/www_incremental_backup.sh +#/opt/bin/tibo_incremental_backup.sh +#/opt/bin/bin_incremental_backup.sh +/opt/bin/backup.sh + +# +# Garder une trace des fichiers modifiés +# +#/opt/bin/trace_incremental_backup.sh diff --git a/bin/oldies/recup_topisto_backups.sh b/bin/oldies/recup_topisto_backups.sh new file mode 100755 index 0000000..6593b40 --- /dev/null +++ b/bin/oldies/recup_topisto_backups.sh @@ -0,0 +1,35 @@ +#!/bin/bash +ladate=`date +%Y%m%d` +targetdir=/opt/topisto.net +rootdir=/tmp/download_topisto +backdir=$rootdir/$ladate + +param=`echo TTNsY2gxczNkM2sK | base64 -d` +salt=`date +%Y%m%d%H` + +cmd="wget --quiet --user backup --password $param" + +if [ ! -d $targetdir ] +then + mkdir -p $targetdir +fi + +if [ ! -d $backdir ] +then + rm -rf $rootdir/* + rm -rf $targetdir/* + + mkdir -p $backdir + + for fichier in www.tar.bz2 TOPISTO_apps.tar.bz2 + do + code=`echo -n /opt/backup/$fichier$salt | md5sum | awk '{ print $1 }' ` + echo Récupération de $fichier dans $backdir/$fichier via $code + $cmd -O $backdir/$fichier https://www.topisto.net/backup/download.php?file=$code + cd $targetdir + echo Extraction de $backdir/$fichier dans $targetdir + tar xfj $backdir/$fichier + done +else + echo "ALREADY DONE : Downloads from topisto.net" +fi diff --git a/bin/oldies/rsync.sh b/bin/oldies/rsync.sh new file mode 100755 index 0000000..b85831a --- /dev/null +++ b/bin/oldies/rsync.sh @@ -0,0 +1,81 @@ +#! /bin/bash + +# Put exclude patterns into ~/.backup.exclude + +# Setup + +DEST="/opt/backups/rsync/" +SRC="/home/tibo/public_html/" + + +############################################################################# +# +# Copyright (c) 2009-2014, Florian Jung +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# +############################################################################# + + +LC_COLLATE="C" # this script relies on the correct sorting order for [0-9]! + +#remove trailing slashes +DEST=$(echo "$DEST" | sed -e 's|\(.*\)/$|\1|') +SRC=$(echo "$SRC" | sed -e 's|\(.*\)/$|\1|') + +cd "$DEST" +# get last and current backup names +LAST="$(echo [0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9] | sed -e 's|.* \([0-9]*\)$|\1|')" +NOW=$(date +%Y%m%d) + +if echo $LAST | grep '\['; then + echo "no previous backup found. creating initial backup..." + LINKDEST="" +else + echo "last backup was: $LAST" + LINKDEST="--link-dest=../$LAST/" +fi + +echo "today's backup is: $NOW" + +if [ $LAST = $NOW ]; then + echo "ERROR: you already did a backup today" +else + FOO=''; + if [ x$1 != x ]; then FOO='-n'; fi +# do the backup +# echo rsync -a -v -x $FOO --hard-links "$LINKDEST" --exclude-from="$SRC/.backup.exclude" "$SRC/" "$DEST/$NOW/" +# rsync -a -v -x $FOO --hard-links "$LINKDEST" --exclude-from="$SRC/.backup.exclude" "$SRC/" "$DEST/$NOW/" + echo rsync -a -v -x $FOO --hard-links "$LINKDEST" --exclude-from="$SRC/.backup.exclude" "$SRC/" "$DEST/$NOW/" + rsync -a -v -x $FOO --hard-links "$LINKDEST" "$SRC/" "$DEST/$NOW/" + echo +# Disable the next 4 lines at your option + echo -n "Today's backup took " + du -sh "$LAST" "$NOW" | tail -n 1 | sed -e "s,$NOW,," + echo -n "Total disk usage by backups: " + du -sh . | sed -e 's,\([kMGT]\).*,\1,' + echo done! + echo +fi diff --git a/bin/oldies/test_recup_topisto_backups.sh b/bin/oldies/test_recup_topisto_backups.sh new file mode 100755 index 0000000..caaa520 --- /dev/null +++ b/bin/oldies/test_recup_topisto_backups.sh @@ -0,0 +1,20 @@ +#!/bin/bash +targetdir=/opt/backups/topisto.net +tmpdir=/tmp/topisto.net +rootdir=/tmp/download_topisto +ladate=`date +%Y%m%d` +param=`echo TTNsY2gxczNkM2sK | base64 -d` +cmd="wget --quiet --user backup --password $param" +backdir=$rootdir/$ladate +salt=`date +%Y%m%d%H` + +for fichier in Twitterbot.tar.bz2 www.tar.bz2 +do + code=`echo -n /opt/backup/$fichier$salt | md5sum | awk '{ print $1 }' ` + echo Récupération de $fichier $code + $cmd -O /tmp/$fichier https://www.topisto.net/backup/download.php?file=$code +done + +ls -lhtr /tmp/*.tar.bz2 +rm -f /tmp/*.tar.bz2 + diff --git a/bin/oldies/tibo_TOPISTO_incremental_backup.sh b/bin/oldies/tibo_TOPISTO_incremental_backup.sh new file mode 100755 index 0000000..d582c0a --- /dev/null +++ b/bin/oldies/tibo_TOPISTO_incremental_backup.sh @@ -0,0 +1,7 @@ +#!/bin/bash +export TARGET=/opt/backups/X11952/TOPISTO +export SOURCE=/home/tibo/TOPISTO +export LADATE=`date +%Y%m%d` +export HIER=`date +%Y%m%d -d "1 day ago"` + +/opt/bin/incremental_backup.sh diff --git a/bin/oldies/tibo_incremental_backup.sh b/bin/oldies/tibo_incremental_backup.sh new file mode 100755 index 0000000..70585fd --- /dev/null +++ b/bin/oldies/tibo_incremental_backup.sh @@ -0,0 +1,9 @@ +#!/bin/bash +export TARGET=/opt/backups/X11952 +export SOURCE=/home/tibo +export LADATE=`date +%Y%m%d` +export HIER=`date +%Y%m%d -d "1 day ago"` + +export EXCLUDED_DIRS="--exclude-from /home/tibo/backups/rsync_excluded_dirs.txt" + +/opt/bin/incremental_backup.sh diff --git a/bin/oldies/tibo_public_html_incremental_backup.sh b/bin/oldies/tibo_public_html_incremental_backup.sh new file mode 100755 index 0000000..846761a --- /dev/null +++ b/bin/oldies/tibo_public_html_incremental_backup.sh @@ -0,0 +1,7 @@ +#!/bin/bash +export TARGET=/opt/backups/X11952/tibo_public_html +export SOURCE=/home/tibo/public_html +export LADATE=`date +%Y%m%d` +export HIER=`date +%Y%m%d -d "1 day ago"` + +/opt/bin/incremental_backup.sh diff --git a/bin/oldies/topisto_incremental_backup.sh b/bin/oldies/topisto_incremental_backup.sh new file mode 100755 index 0000000..13dffa9 --- /dev/null +++ b/bin/oldies/topisto_incremental_backup.sh @@ -0,0 +1,12 @@ +#!/bin/bash +export TARGET=/opt/topisto.net +export LADATE=`date +%Y%m%d` +export HIER=`date +%Y%m%d -d "1 day ago"` + +export SOURCE=/tmp/topisto.net/www + +/opt/bin/incremental_backup.sh + +export SOURCE=/tmp/topisto.net/TOPISTO + +/opt/bin/incremental_backup.sh diff --git a/bin/oldies/trace_incremental_backup.sh b/bin/oldies/trace_incremental_backup.sh new file mode 100755 index 0000000..fa9a305 --- /dev/null +++ b/bin/oldies/trace_incremental_backup.sh @@ -0,0 +1,18 @@ +#!/bin/bash +export LADATE=`date +%Y%m%d` +export HIER=`date +%Y%m%d -d "1 day ago"` + +export LEJOUR=`date +%d` +export TODELETE=`date +%Y%m%d -d "3 months ago"` + +# Garder une trace des fichiers qui ont changé +# c'est à dire ceux qui ne sont pas des liens +find /opt/backups/*/$LADATE -type f -printf '%n %p\n' | awk '$1 == 1{print $2}' > /opt/backups/trace_$LADATE.log + + +if [ "$LEJOUR" != "01" ] +then + echo TO DELETE $TODELETE + rm -f /opt/backups/trace_$TODELETE.log + rm -rf /opt/backups/*/$TODELETE +fi diff --git a/bin/oldies/twitterbot_incremental_backup.sh b/bin/oldies/twitterbot_incremental_backup.sh new file mode 100755 index 0000000..20b78c0 --- /dev/null +++ b/bin/oldies/twitterbot_incremental_backup.sh @@ -0,0 +1,7 @@ +#!/bin/bash +export TARGET=/opt/backups/Twitterbot +export SOURCE=/opt/Twitterbot +export LADATE=`date +%Y%m%d` +export HIER=`date +%Y%m%d -d "1 day ago"` + +/opt/bin/incremental_backup.sh diff --git a/bin/oldies/www_incremental_backup.sh b/bin/oldies/www_incremental_backup.sh new file mode 100755 index 0000000..e59b2a4 --- /dev/null +++ b/bin/oldies/www_incremental_backup.sh @@ -0,0 +1,7 @@ +#!/bin/bash +export TARGET=/opt/backups/X11952 +export SOURCE=/var/www/html +export LADATE=`date +%Y%m%d` +export HIER=`date +%Y%m%d -d "1 day ago"` + +/opt/bin/incremental_backup.sh diff --git a/bin/r_backup.sh b/bin/r_backup.sh new file mode 100755 index 0000000..0dc904b --- /dev/null +++ b/bin/r_backup.sh @@ -0,0 +1,42 @@ +#!/bin/bash +export TARGET=/opt/backups +export LADATE=`date +%Y%m%d` +export TODELETE=`date +%Y%m%d -d "3 months ago"` + +SCRIPT=`basename $0 .sh` +FLAGFILE=$TARGET/flags/$SCRIPT.$LADATE + +if [ -f $FLAGFILE ] +then + exit 0 +fi + +if [ ! -f $TARGET/data/liste.txt ] +then + echo No $TARGET/data/liste.txt found ... + exit 1 +fi + +if [ ! -f $TARGET/data/signatures.sqlite ] +then + echo No $TARGET/data/signatures.sqlite found ... + exit 1 +fi + +HEURE=`date +%H:%M` +echo "--- $HEURE - BEGIN" >> $FLAGFILE + +for SOURCE in `grep -v "#" $TARGET/data/liste.txt` +do + HEURE=`date +%H:%M` + echo "--- $HEURE - $SOURCE" >> $FLAGFILE + php $TARGET/scripts/backup.php $SOURCE $TARGET/data +done + +HEURE=`date +%H:%M` +echo "--- $HEURE - END" >> $FLAGFILE + +sqlite3 $TARGET/data/signatures.sqlite 'VACUUM' +cp $TARGET/data/signatures.sqlite $TARGET/data/$LADATE/signatures.sqlite + +sqlite3 $TARGET/data/signatures.sqlite 'select substr(filename,0,9) as date, count(*) as nbre, sum(filesize) as size from signatures group by substr(filename,0,9)' >> $FLAGFILE diff --git a/data/liste.txt b/data/liste.txt new file mode 100644 index 0000000..bae944f --- /dev/null +++ b/data/liste.txt @@ -0,0 +1,4 @@ +/opt/bin +/opt/topisto.net +/var/www +/home/tibo diff --git a/scripts/backup.php b/scripts/backup.php new file mode 100644 index 0000000..387ce67 --- /dev/null +++ b/scripts/backup.php @@ -0,0 +1,177 @@ +prepare($insert); + + $update = "UPDATE signatures SET filename = :filename , filesize = :filesize WHERE sha1 = :sha1"; + $stmt2 = $file_db->prepare($update); + + $delete = "DELETE FROM signatures WHERE sha1 = :sha1"; + $stmt3 = $file_db->prepare($delete); + + my_echo_with_eol("Searching SHA1 in database for $basename"); + $result = $file_db->query("SELECT filename, filesize FROM signatures WHERE sha1 = '$sha1_fichier'"); + if ($result !== FALSE) + { + foreach($result as $f) + { + my_echo_with_eol("Found ".$f['filename']); + if ($f['filesize'] != $filesize) my_echo_with_eol("Oups, database and file filesize differ !"); + $in = $destination.'/'.$f['filename']; + $out = $archive; + if (file_exists($in)) link($in, $out); + if (file_exists("$in.gz")) link("$in.gz", "$out.gz"); + if (file_exists($out)||file_exists("$out.gz")) + { + $stmt2->execute(array(':sha1' => $sha1_fichier, ':filename' => $filename, ':filesize' => $filesize)); + $flag = true; + my_echo_with_eol("Update database filename : $filename"); + } + + if (!$flag) + { + my_echo_with_eol("Delete database sha1 : $sha1_fichier"); + $stmt3->execute(array(':sha1' => $sha1_fichier)); + } + } + } + if (!$flag) + { + my_echo_with_eol("Not Found $basename"); + + // Les formats compressés sont juste copiés + $ext = pathinfo($fichier, PATHINFO_EXTENSION); + if ($ext == "avi") copy($fichier, $archive); + if ($ext == "mkv") copy($fichier, $archive); + if ($ext == "mp4") copy($fichier, $archive); + if ($ext == "mp3") copy($fichier, $archive); + + if ($ext == "bz2") copy($fichier, $archive); + if ($ext == "gz") copy($fichier, $archive); + if ($ext == "zip") copy($fichier, $archive); + + if ($ext == "iso") copy($fichier, $archive); + + if (!file_exists($archive)) + { + my_echo_with_eol("Compress $basename"); + gzCompressFile($fichier, "$archive.gz"); + } else my_echo_with_eol("Copy $basename"); + + my_echo_with_eol("Insert database filename : $filename"); + $stmt1->execute(array(':sha1' => $sha1_fichier, ':filename' => $filename, ':filesize' => $filesize)); + } + my_echo_with_eol("--------- "); + return TRUE; +} + +function listeDossier($file_db, $dossier, $destination) // Fonction qui liste un dossier de façon récursive +{ + $ladate = date('Ymd'); + + if (file_exists("$destination/$ladate/$dossier")) + { + my_echo_with_eol('BACKUP already done'); + return true; + } + + if (is_dir($dossier)) + { + if (!file_exists("$destination/$ladate")) mkdir("$destination/$ladate"); + $path = explode("/", $dossier); + $init_path = "$destination/$ladate"; + foreach($path as $p) + { + $init_path .= '/'.$p; + if (!file_exists($init_path)) mkdir($init_path); + } + my_echo_with_eol("BACKUP OF $dossier"); + if($dossierOuvert=opendir($dossier)) + { + $n = 0; + while(($fichier=readdir($dossierOuvert))!== false) + { + // if ($n++ == 8) break; + + if ($fichier[0] == '.') continue; + + if(is_dir("$dossier/$fichier")) + listeDossier($file_db, "$dossier/$fichier", $destination); + else + fileBackup($file_db, "$dossier/$fichier", $destination); + } + } + } + else + { + my_echo_with_eol("Erreur, le paramètre précisé dans la fonction n'est pas un dossier!"); + } +} + +$source = $argv[1]; +$backup = $argv[2]; +$signatures=$backup.'/signatures.sqlite'; + +if (!file_exists($source)) die("$source NOT FOUND !"); +if (!is_dir($backup)) die("$backup NOT FOUND !"); +if (!file_exists($signatures)) die("$signatures NOT FOUND !"); + +$connexion = new PDO('sqlite:'.$signatures); +my_echo_with_eol("DATABASE IS OPEN"); +if(is_dir($backup)) + listeDossier($connexion, $source, $backup); +else + fileBackup($connexion, $source, $backup); +$connexion = null; +my_echo_with_eol("DATABASE IS CLOSED"); + +?> diff --git a/scripts/controle.php b/scripts/controle.php new file mode 100644 index 0000000..455a161 --- /dev/null +++ b/scripts/controle.php @@ -0,0 +1,32 @@ +query("SELECT sha1, filename, filesize FROM signatures"); +if ($result !== FALSE) + foreach($result as $f) + { + $flag = FALSE; + $fichier = $backup.'/'.$f['filename']; + + if (file_exists($fichier)) $flag = TRUE; + if (file_exists("$fichier.gz")) $flag = TRUE; + + if (!$flag) + { + echo $fichier.' inexistant !'.PHP_EOL; + $global_flag = FALSE; + } + } +$connexion = null; + +if ($global_flag) echo "Control signatures OK".PHP_EOL; +else echo "Control signatures NOK".PHP_EOL; + +?> diff --git a/scripts/sigSHA1.sqlite.template b/scripts/sigSHA1.sqlite.template new file mode 100644 index 0000000..3bc7814 Binary files /dev/null and b/scripts/sigSHA1.sqlite.template differ