Merge branch 'FixSnapper' into 'master'

Fix snapper

See merge request readynas-scripts/share_lun_ui_fix.sh!11

.gitlab-ci.yml

@@ -0,0 +1,4 @@
+shellcheck:
+  stage: build
+  script:
+    - /usr/bin/shellcheck -e SC2164,SC2086 share_lun_ui_fix.sh

README.md

@@ -1,11 +1,12 @@
 # Share/LUN UI Fix Script
 
 This script finds shares that were accidentally broken in the UI and fixes it.
+Additionally, performs migration of created folders in the roots of volumes to btrfs subvolume. 
 
 
 ### Prerequisites
 
-* You need to be on ReadyNAS OS 6.9.4 or newer. This script contains configuration file data for 6.9.4 that may or may not work with older versions.
+* You should use this script with ReadyNAS OS 6.10.0 or newer. This script contains configuration file data for 6.10.0 that may or may not work with older versions of ReadyNASOS.
 * Either the ReadyNAS needs internet access or you need to copy/paste the data into a file on the NAS via SSH.
 ** For internet access, ensure you can reach the internet and DNS is working properly.
 * This does not work in `Tech Support mode`. You must be in `Normal mode` and `readynasd` needs to be running. 
@@ -44,58 +45,61 @@ root@readynasos:#/ bash -x /root/share_lun_ui_fix.sh
 ### Standard Output
 ```
 root@rn524x:~# curl -s https://gitlab.codycook.us/readynas-scripts/share_lun_ui_fix.sh/raw/master/share_lun_ui_fix.sh | bash
-===============================
+===================================
-Share and iSCSI Rebuild v6.9.4
+* Share and iSCSI Rebuild v6.10.0 *
-===============================
+===================================
-NOTE: This version of readynasd creates share backups in /var/backups/shares.
+= [Info] Share config backups folder exist (/var/backups/shares)
-      When possible, you should try and restore a valid backup config from this directory.
+== [Task] Backing up existing share config backup folder
-      This share config gets rebuilt every time readynasd restarts. Because of this, we'll make a backup of the existing share backups before proceeding.
+=== [Info] Completed share config backup folder.
-Generating backup config...... done! Backup complete!
+=== [Task] Listing backup folder for a quick analysis
-=============================================
+2019-04-01 22:56:12.198942551 -0700 71680 /var/backups/shares/shares.tar.0
-====== Listing Available Share Backups ======
+2019-04-01 22:50:02.066740050 -0700 2058 /var/backups/shares/shares.tar.1.gz
-=============================================
+2019-04-01 21:40:46.250729403 -0700 2012 /var/backups/shares/shares.tar.2.gz
-total 1.3M
+2019-04-01 15:30:49.968039883 -0700 1754 /var/backups/shares/shares.tar.3.gz
-drwxr-x--- 1 root root  144 Oct  2 04:20 .
+2019-04-01 15:28:44.033709334 -0700 1753 /var/backups/shares/shares.tar.4.gz
-drwxr-xr-x 1 root root  866 Oct  2 10:31 ..
+==== [Info] Review backup folder listing and decide to continue or restore.
--rw-r--r-- 1 root root 750K Oct  2 04:20 shares.tar.0
+==== [Decide] To cancel and manually restore from an above config, CTRL-C now... Otherwise, continuing in 5... 4... 3... 2... 1... 0...
--rw-r--r-- 1 root root 124K Oct  1 04:20 shares.tar.1.gz
+= [Task] Discovering volumes
--rw-r--r-- 1 root root 124K Sep 30 04:20 shares.tar.2.gz
+== [Info] 1 volume(s) found.
--rw-r--r-- 1 root root 124K Sep 29 04:20 shares.tar.3.gz
+== [Task] Investigating volume /data...
--rw-r--r-- 1 root root 124K Sep 28 04:20 shares.tar.4.gz
+=== [Info] Number of items found : 6
-=============================================
+=== [Info] Items found: Documents ImFake LUN1 Music Pictures Videos
-See the last modified dates and determine if you can restore from that config or not instead of using this script.
+=== [Task] Reviewing Documents
-It is best to pick a date before the shares disappeared from readynasd.
+=== [Info] Found share at /data/Documents
-=============================================
+=== [Task] Recreating share configs.
-If you want to cancel and manually restore from one of these configs, CTRL-C now...
+==== [Info] Done with Share /data/Documents.
-Otherwise, continuing in 10... 9... 8... 7... 6... 5... 4... 3... 2... 1... 0...
+=== [Task] Reviewing ImFake
-Querying readynasd for list of volumes...
+=== [Task] Adding /data/ImFake to subvolume migration list.
-4 volume(s) found.
+=== [Task] Reviewing LUN1
-Beginning in 3... 2... 1...
+=== [Task] Recreating config for LUN at /data/LUN1..
-===============================
+==== [Info] done with LUN /data/LUN1.
-=== All right!  Time to go! ===
+=== [Task] Reviewing Music
-===============================
+=== [Info] Found share at /data/Music
-= Investigating volume /vol1...
+=== [Task] Recreating share configs.
-= Objects found: fromharry
+==== [Info] Done with Share /data/Music.
-== Reviewing fromharry
+=== [Task] Reviewing Pictures
-=== Found share at /vol1/fromharry... recreating share configs.... done with /vol1/fromharry.
+=== [Info] Found share at /data/Pictures
-= done with /vol1
+=== [Task] Recreating share configs.
-= Investigating volume /vol2...
+==== [Info] Done with Share /data/Pictures.
-= Objects found: luntest
+=== [Task] Reviewing Videos
-== Reviewing luntest
+=== [Info] Found share at /data/Videos
-=== Found LUN at /vol2/luntest... recreating iSCSI configs.... done with /vol2/luntest.
+=== [Task] Recreating share configs.
-= done with /vol2
+==== [Info] Done with Share /data/Videos.
-= Investigating volume /vol3...
+== [Task] Subvolume Migrations required: 1
-No shares or LUNs found.
+== [Info] Subvolume Migrations list:  ImFake
-= done with /vol3
+=== [Warning] Do not interrupt this process. Doing so may result in an incomplete migration.
-= Investigating volume /vol4...
+=== [Decide] If you want to terminate now, CTRL-C...Continuing in 5... 4... 3... 2... 1... 0...
-= Objects found: outrunner
+=== [Task] Subvolume Migrating /data/ImFake
-== Reviewing outrunner
+=== [Task] Creating replacement btrfs subvolume
-=== Found share at /vol4/outrunner... recreating share configs.... done with /vol4/outrunner.
+==== [Task] Migrating data to the new subvolume
-= done with /vol4
+===== [Success] migrating /data/ImFake to a btrfs subvolume.
-*** Restarting readynasd to finish changes... ***
+=== [Info] Found share at /data/ImFake
-=================================================
+=== [Task] Recreating share configs.
-| Completed share creation! We were able to     |
+=== [Info] done with /data/ImFake.
-| rebuild 2 shares and 1 LUNs.                  |
+= [Info] done with /data
-=================================================
+= [Task] Restarting readynasd to apply changes.
+[Stats] Rebuilt Share configs: 5
+[Stats] Rebuilt LUN configs: 1
+[Stats] Migrated folders to subvolumes: 1
 root@rn524x:~#
 ```

share_lun_ui_fix.sh

@@ -2,18 +2,36 @@
 # share fix
 # use this script to rebuild default configs for shares if the ._share becomes
 # broken for whatever reason. 
-unset readynasd number x volume found iscsi shares
+unset readynasd number x volume found iscsi shares invalid verbosity volumes
 found=0
 shares=0
 iscsi=0
+migrated=0
+mvcmd="mv"
+################
+### commands ###
+################
+
+backupsnappershots() {
+    echo "==== [Task] Backing up existing Snapper configs"
+    tar -cf "/var/backups/backup_snapper_configs.$(date +%s).tar" /etc/snapper/configs 2>/dev/null 
+    if [[ $? ]]; then
+        echo "===== [Info] Success at backing up snapper configs"
+        echo "===== [Task] Removing Snapper Configs"
+        rm /etc/snapper/configs/* 2>/dev/null
+    else
+        echo "===== [Fail] Couldn't backup configs. Please debug"
+    fi
+}
 share_create() {
 	found=$((found+1))
 	shares=$((shares+1))
 	path="/$volume/._share/$item"
-        echo -n "=== Found share at /$volume/$item... recreating share configs...".
+        echo "=== [Info] Found share at /$volume/$item"
+        echo "=== [Task] Recreating share configs."
         mkdir -p "$path"
         echo "share" > "$path"/datasettype.conf
-        echo "0,0" > "$path"/recycle.conf
+        echo "0,25000" > "$path"/recycle.conf
         echo "#dashboard:available=0 \"/$volume/$item\" 127.0.0.1(insecure,insecure_locks,no_subtree_check,crossmnt,anonuid=99,anongid=99,root_squash,ro,async)" > "$path"/nfs.conf
         echo "" > "$path"/snapdir.conf
         echo "1 0 0 * * * 0" > "$path"/snapshot.conf
@@ -48,7 +66,7 @@ refquota = 0
 EOF
         cat > "$path"/ftp.conf << EOF
 available       0
-default_access  disabled
+default_access  writeable
 mask
 fmask
 dmask
@@ -62,62 +80,123 @@ EOF
         cat > "$path"/samba.conf << EOF
 [$item]
   path = /$volume/$item
-  comment = ""
+  comment = "$item"
   spotlight = 0
   guest ok = 1
+  force create mode = 0666
+  create mask = 0666
+  force directory mode = 1777
+  directory mask = 1777
   admin users = +admin
   writeable = 1
   follow symlinks = 1
 EOF
-	cd "$path"
+
+        cat > /etc/snapper/configs/$((shares-1)) << EOF
+
+# subvolume to snapshot
+SUBVOLUME="/$volume/$item"
+
+# filesystem type
+FSTYPE="btrfs"
+
+
+# btrfs qgroup for space aware cleanup algorithms
+QGROUP=""
+
+
+# fraction of the filesystems space the snapshots may use
+SPACE_LIMIT="0.5"
+
+
+# users and groups allowed to work with config
+ALLOW_USERS="guest"
+ALLOW_GROUPS="admin"
+
+# sync users and groups from ALLOW_USERS and ALLOW_GROUPS to .snapshots
+# directory
+SYNC_ACL="no"
+
+
+# start comparing pre- and post-snapshot in background after creating
+# post-snapshot
+BACKGROUND_COMPARISON="yes"
+
+
+# run daily number cleanup
+NUMBER_CLEANUP="yes"
+
+# limit for number cleanup
+NUMBER_MIN_AGE="1800"
+NUMBER_LIMIT="50"
+NUMBER_LIMIT_IMPORTANT="10"
+
+
+# create hourly snapshots
+TIMELINE_CREATE="yes"
+
+# cleanup hourly snapshots after some time
+TIMELINE_CLEANUP="yes"
+
+# limits for timeline cleanup
+TIMELINE_MIN_AGE="1800"
+TIMELINE_LIMIT_HOURLY="10"
+TIMELINE_LIMIT_DAILY="10"
+TIMELINE_LIMIT_WEEKLY="0"
+TIMELINE_LIMIT_MONTHLY="10"
+TIMELINE_LIMIT_YEARLY="10"
+
+
+# cleanup empty pre-post-pairs
+EMPTY_PRE_POST_CLEANUP="yes"
+
+# limits for empty pre-post-pair cleanup
+EMPTY_PRE_POST_MIN_AGE="1800"
+
+SNAPSHOT_HWM="13"
+EOF
+    cd "$path"
         chown root.root datasettype.conf recycle.conf snapdir.conf snapshot.conf
         chown admin.admin afp.conf fs.conf ftp.conf nfs.conf samba.conf custom_snapshot_management.conf custom_snapshot_schedule.conf
         chmod 644 datasettype.conf recycle.conf snapdir.conf snapshot.conf afp.conf fs.conf ftp.conf nfs.conf samba.conf custom_snapshot_management.conf custom_snapshot_schedule.conf
-        echo " done with /$volume/$item."
+        chmod 640 /etc/snapper/configs/$((shares-1))
+        echo "==== [Info] Done with Share /$volume/$item."
 }
 
 check_existing_backups() {
     
     if [ -d "/var/backups/shares/" ]; then
-        echo "NOTE: This version of readynasd creates share backups in /var/backups/shares."
+        echo "= [Info] Share config backups folder exist (/var/backups/shares)"
-        echo "      When possible, you should try and restore a valid backup config from this directory."
+        echo "== [Task] Backing up existing share config backup folder"
-        echo "      This share config gets rebuilt every time readynasd restarts. Because of this, we'll make a backup of the existing share backups before proceeding."
+        tar -cf "/var/backups/backup_share_configs.$(date +%s).tar" /var/backups/shares 2>/dev/null 
-        sleep 3
+        echo "=== [Info] Completed share config backup folder."
-        echo -n "Generating backup config..." && sleep 1 && tar -cf "/var/backups/backup_share_configs.$(date +%s).tar" /var/backups/shares 2>/dev/null && sleep 1 && echo "... done! Backup complete!"
-        sleep 3
-        echo "============================================="
-        echo "====== Listing Available Share Backups ======"
-        echo "============================================="
-        sleep 2
-        ls -lah /var/backups/shares/
         sleep 1
-        echo "============================================="
+        echo "=== [Task] Listing backup folder for a quick analysis"
-        echo "See the last modified dates and determine if you can restore from that config or not instead of using this script."
+        stat -c "%y %s %n" /var/backups/shares/*
-        echo "It is best to pick a date before the shares disappeared from readynasd."
+        echo "==== [Info] Review backup folder listing and decide to continue or restore."
-        echo "============================================="
         sleep 5
-        echo "If you want to cancel and manually restore from one of these configs, CTRL-C now..."
+        echo -n "==== [Decide] To cancel and manually restore from an above config, CTRL-C now... Otherwise, continuing in 5..."; 
-        echo -n "Otherwise, continuing in 10..."; sleep 1; echo -n " 9..."; sleep 1; echo -n  " 8..."; sleep 1; echo -n  " 7..."; sleep 1; echo -n  " 6..."; sleep 1; echo -n  " 5..."; sleep 1; echo -n  " 4..."; sleep  1; echo -n " 3..."; sleep 1; echo -n  " 2..."; sleep 1; echo -n  " 1..."; sleep 1 ; echo " 0..."; sleep 1
+        sleep 1; 
+        echo -n  " 4..."; 
+        sleep  1; 
+        echo -n " 3..."; 
+        sleep 1; 
+        echo -n  " 2..."; 
+        sleep 1; 
+        echo -n  " 1..."; 
+        sleep 1 ; 
+        echo " 0..."; 
+        sleep 1
     fi
 }
 
-subv_checker() {
-# Check to see if subv list matches actual directory folders
-# This might be because user created folders in the root that are not btrfs subvolumes
-# or the user moved a subv from the root to another subvolume (not supported in OS 6)
-
-# Check the count
-# Objects counting
-count_btrfs=$(btrfs subv list /data | egrep -v ".snapshots|.apps|.vault|._share|.timemachine|.purge|home\/|home$" | awk '{print $9}' | wc -l)
-}
-
 iscsi_create() {
 	iscsi=$((iscsi+1))
-        found=$((found+1))
+    found=$((found+1))
-        path="/$volume/._share/$item"
+    path="/$volume/._share/$item"
-        echo -ne "=== Found LUN at /$volume/$item... recreating iSCSI configs...".
+    echo  "=== [Task] Recreating config for LUN at /$volume/$item.."
 	mkdir -p "$path"
-        cat > "$path"/iscsi.conf << EOF
+    cat > "$path"/iscsi.conf << EOF
 comment
 alert_threshold 80
 EOF
@@ -139,72 +218,112 @@ EOF
 ........................
 EOF
     echo "0 0 0 * 0 0 0" > "$path"/snapshot.conf
-	echo " done with /$volume/$item."
+	echo "==== [Info] done with LUN /$volume/$item."
 }
 
 backup_configs() {
-cd "/$volume/._share/"
+cd "/$volume/._share/" 
-tar -cf "/var/backups/$volume_sharelunconfig.$(date +%s).tar" ./* 2>/dev/null
+tar -cf "/var/backups/$volume.sharelunconfig.$(date +%s).tar" ./* 2>/dev/null
 }
 
-echo "==============================="
+checktype() {
-echo "Share and iSCSI Rebuild v6.9.4"
+        if ls -QdA /"$volume"/"$item"/.iscsi >/dev/null 2>&1; then
-echo "==============================="
+                iscsi_create
+        else
+                share_create
+        fi
+}
+###################
+### end command ###
+###################
+
+
+echo "==================================="
+echo "* Share and iSCSI Rebuild v6.10.0 *"
+echo "==================================="
 
 check_existing_backups
 
-echo "Querying readynasd for list of volumes..."
+echo "= [Task] Discovering volumes"
-
+volumes=$(rn_nml -g volumes 2>/dev/null)  # find if readynasd is running
-x=$(rn_nml -g volumes 2>/dev/null)  # find if readynasd is running
-
 if [[ $? -eq 0 ]]; then # if readynasd was successful
-	x=$(echo "$x"| grep "resource-id" | sed 's/^.*id="//;s/".*//;') # then parse it
+	volumes=$(echo "$volumes"| grep "resource-id" | sed 's/^.*id="//;s/".*//;') # then parse it
 	readynasd=1
 elif [[ $? -eq 1 ]] ; then # but if it wasn't
-	echo "readynasd not available; doing alternative lookup for shares"     # inform intent
+	echo "== [Info] readynasd not available; doing alternative lookup for shares"     # inform intent
-	x=$(find / -maxdepth 2 -name "._share" | sed 's/^\///g;s/\/.*//')       # create new list
+	volumes=$(find / -maxdepth 2 -name "._share" | sed 's/^\///g;s/\/.*//')       # create new list
 fi
-
+number=$(echo "$volumes" | wc -w)
-number=$(echo "$x" | wc -w)
-
 if [[ ! $number ]]; then 
-	echo "No volumes found!"
+	echo "== [Error] No volumes found."
 	exit 1
 else
-	echo "$number volume(s) found."
+	echo "== [Info] $number volume(s) found."
-	echo -n "Beginning in 3..."; sleep 1; echo -n " 2..."; sleep 1; echo " 1..."; sleep 1
+	backupsnappershots
-	echo "==============================="
+	for volume in $volumes; do
-	echo "=== All right!  Time to go! ==="
+		echo "== [Task] Investigating volume /$volume..."
-	echo "==============================="
+		objects=$(cd /"$volume"; ls -dA ./*/ 2>/dev/null)
-	for volume in $x; do
-		echo "= Investigating volume /$volume..."
-		objects=$(cd /"$volume"; ls -dA */ 2>/dev/null)
 		if [[ $? == 2 ]]; then
-			echo "No shares or LUNs found."
+			echo "=== [Error] No shares or LUNs found."
 		else
-			objects=$(echo "$objects" | egrep -v "^.apps\/$|^home\/$|^.purge\/$|^._share\/$|^.timemachine\/$|^.vault\/$|^.TemporaryItems\/$" | sed 's/\/$//')
+			objects=$(echo "$objects" |egrep -v "^.apps\/$|^\.\/home\/$|^.purge\/$|^._share\/$|^.timemachine\/$|^.vault\/$|^.TemporaryItems\/$" | sed 's/\/$//;s/\.\///')
-		        echo "= Objects found:" $objects
+            echo "=== [Info] Number of items found : $(echo $objects | wc -w)"
-                        subv_checker
+		    echo "=== [Info] Items found:" $objects
-			backup_configs
+	    	backup_configs
-		        for item in $objects; do
+		    for item in $objects; do
-		                echo "== Reviewing $item"
+                echo "=== [Task] Reviewing $item"         
-		                if ls -QdA /"$volume"/"$item"/.iscsi >/dev/null 2>&1; then
+                if ! btrfs subv show /"$volume"/"$item" &>/dev/null; then
-					iscsi_create
+                    echo "=== [Task] Adding /$volume/$item to subvolume migration list."
-				else
+                    invalid="$invalid $item"
-	                        	share_create
+                else
-		                fi
+                    checktype
-			done
+                fi
+		    done
 		fi
-		echo "= done with /$volume"
+        if [[ $invalid ]]; then
-	done
+            invalidnumber=$(echo "$invalid" | wc -w)
+            echo "== [Task] Subvolume Migrations required: $invalidnumber"
+            echo "== [Info] Subvolume Migrations list: $invalid"
+            echo "=== [Warning] Do not interrupt this process. Doing so may result in an incomplete migration."
+            sleep 3
+            echo -n "=== [Decide] If you want to terminate now, CTRL-C..." && echo -n "Continuing in 5..."; sleep 1; echo -n  " 4..."; sleep  1; echo -n " 3..."; sleep 1; echo -n  " 2..."; sleep 1; echo -n  " 1..."; sleep 1 ; echo " 0..."; sleep 1
+            for item in $invalid; do
+                echo "=== [Task] Subvolume Migrating /$volume/$item"
+                $mvcmd /"$volume"/"$item" /"$volume"/"$item".fix
+                if [[ ! $? ]]; then
+                    echo "==== [Fail] when renaming folder."
+                elif [[ $? ]]; then
+                    echo "=== [Task] Creating replacement btrfs subvolume"
+                    btrfs subvolume create /"$volume"/"$item" &>/dev/null
+                    if [[ $? ]]; then
+                        echo "==== [Task] Migrating data to the new subvolume"
+                        $mvcmd /"$volume"/"$item".fix/* /"$volume"/"$item".fix/.* /"$volume"/"$item" &>/dev/null
+                        if [[ $? ]]; then
+                            echo "===== [Success] migrating /$volume/$item to a btrfs subvolume."
+                            rmdir /$volume/$item.fix
+                            migrated=$((migrated+1))
+                            checktype
+                        else
+                            echo "===== [Error] Re-run in debug mode for more details."
+                        fi
+                    fi
+                else
+                    echo "==== [Fail] when creating btrfs subvolume."
+                fi
+            done
+    
+        fi
+		   echo "= [Info] done with /$volume"
+                unset invalid invalidnumber
+        done
+
 	if [[ $found -gt 0 ]] && [[ $readynasd ]] ; then
-		echo "*** Restarting readynasd to finish changes... ***"
+		echo "= [Task] Restarting readynasd to apply changes."
 		systemctl restart readynasd
 	else
-		echo "Not starting readynasd because it returned a bad message earlier; restart manually."
+		echo "= [Info] Not starting readynasd because it returned a bad message earlier; restart as needed."
 	fi
-	echo "================================================="
+	echo "[Stats] Rebuilt Share configs: $shares"
-	echo "| Completed share creation! We were able to     |"
+	echo "[Stats] Rebuilt LUN configs: $iscsi"
-	echo "| rebuild $shares shares and $iscsi LUNs.                  |"
+	echo "[Stats] Migrated folders to subvolumes: $migrated"
-	echo "================================================="
 fi