diff --git a/sysutils/nextcloud-backup/src/opnsense/mvc/app/library/OPNsense/Backup/Nextcloud.php b/sysutils/nextcloud-backup/src/opnsense/mvc/app/library/OPNsense/Backup/Nextcloud.php index 58956a44c..0dc122128 100644 --- a/sysutils/nextcloud-backup/src/opnsense/mvc/app/library/OPNsense/Backup/Nextcloud.php +++ b/sysutils/nextcloud-backup/src/opnsense/mvc/app/library/OPNsense/Backup/Nextcloud.php @@ -98,6 +98,20 @@ class Nextcloud extends Base implements IBackupProvider "help" => gettext("Create subdirectory under backupdir for this host"), "value" => null ), + array( + "name" => "numdays", + "type" => "text", + "label" => gettext("Number of days worth of backups to keep"), + "help" => gettext("This works in collaboration with Number of backups below, the one with the oldest/most will win"), + "value" => null + ), + array( + "name" => "numbackups", + "type" => "text", + "label" => gettext("Number of backups to keep"), + "help" => gettext("This works in collaboration with Number of days above, the one with the oldest/most will win"), + "value" => null + ), ); $nextcloud = new NextcloudSettings(); foreach ($fields as &$field) { @@ -167,7 +181,267 @@ class Nextcloud extends Base implements IBackupProvider return 0; } + /** + * backup with strategy 0 (copy everything in /conf/backup/ to $backupdir/) + * @param string $internal_username the returnvalue from $this->getInternalUsername + * @param string $username + * @param string $password + * @param string $url server protocol and hostname + * @param string $backupdir + * @param string $crypto_password + */ + public function backupstrat_zero( + $internal_username, + $username, + $password, + $url, + $backupdir, + $crypto_password + ) { + // Get list of files from local backup system + $local_files = array(); + $tmp_local_files = scandir('/conf/backup/'); + // Remove '.' and '..', skip directories + foreach ($tmp_local_files as $tmp_local_file) { + if ($tmp_local_file === '.' || $tmp_local_file === '..') { + continue; + } + if (!is_file("/conf/backup/".$tmp_local_file)) { + continue; + } + $local_files[] = $tmp_local_file; + } + // Get list of filenames (without path) on remote location + $remote_files = array(); + $tmp_remote_files = $this->listfiles($url, $username, $password, $internal_username, "/$backupdir/", false); + foreach ($tmp_remote_files as $tmp_remote_file) { + $remote_files[] = pathinfo($tmp_remote_file)['basename']; + } + + + $uploaded_files = array(); + + // Loop over each local file, + // see if it's in $remote_files, + // if not, optionally encrypt, and upload + foreach ($local_files as $file_to_upload) { + if (!in_array($file_to_upload, $remote_files)) { + $confdata = file_get_contents("/conf/backup/$file_to_upload"); + if (!empty($crypto_password)) { + $confdata = $this->encrypt($confdata, $crypto_password); + } + try { + $this->upload_file_content( + $url, + $username, + $password, + $internal_username, + $backupdir, + $file_to_upload, + $confdata + ); + $uploaded_files[] = $file_to_upload; + } catch (\Exception $e) { + return $uploaded_files; + } + } + } + return $uploaded_files; + } + + /** + * backup with strategy 1 (copy /conf/config.xml to $backupdir/conf-YYYYMMDD.xml) + * @param string $internal_username the returnvalue from $this->getInternalUsername + * @param string $username + * @param string $password + * @param string url server protocol and hostname + * @param string $backupdir + * @param string $crypto_password + */ + public function backupstrat_one( + $internal_username, + $username, + $password, + $url, + $backupdir, + $crypto_password + ) { + $confdata = file_get_contents('/conf/config.xml'); + $mdate = filemtime('/conf/config.xml'); + $datestring = date('Ymd', $mdate); + $target_filename = 'config-' . $datestring . '.xml'; + // Find the same filename @ remote + $remote_filename = $url . '/remote.php/dav/files/' . $internal_username . '/' . $backupdir . '/' . $target_filename; + $remote_file_date = $this->get_remote_file_lastmodified($remote_filename, $username, $password); + if ($remote_file_date >= $mdate) { + return array(); + } + + // Optionally encrypt + if (!empty($crypto_password)) { + $confdata = $this->encrypt($confdata, $crypto_password); + } + // Finally, upload some data + try { + $this->upload_file_content( + $url, + $username, + $password, + $internal_username, + $backupdir, + $target_filename, + $confdata + ); + return array($backupdir . '/' . $target_filename); + } catch (\Exception $e) { + syslog(LOG_ERR, 'Backup to ' . $url . ' failed: ' . $e); + return array(); + } + } + + /** + * Get timestamp value from filename in list + * @param $filelist array of files in remote location + * @return array($filedata => $filename) + */ + public function get_filelist_dates($filelist) { + // Save as associative array + // key = lastmodified + // value = filename + $files = array(); + foreach ($filelist as $target_filename) { + // Find suggested creation date + // Base this on the filename. Either it is a unix timestamp, or it should be YYYYMMDD + // Either way, it's the part between "config-" and ".xml" + $filestr_no_xml = explode(".xml", $target_filename)[0]; + $filedatestr = intval(explode("-", $filestr_no_xml)[1]); + if (($filedate = strtotime($filedatestr)) === false) { + // Cannot convert string to time.. probably already a unix-timestamp + // Try to convert with date() + $date = date(DATE_ATOM, $filedatestr); + // Then to a UNIX timestamp again + $maybedate = strtotime($date); + if ($maybedate === $filedatestr) { + // They represent the same time, this is good + // Just copy the intval() and be done with this + $filedate = $filedatestr; + } + } + if ($filedate) { + $files[(string)$filedate] = $target_filename; + } else { + syslog(LOG_ERR, "Skipping file " . $target_filename . ", cannot determine date"); + } + } + ksort($files); + return $files; + } + + /** + * housekeeping + * @param $internal_username returnvalue from $this->getInternalUsername + * @param $username + * @param $password + * @param $url protocol and hostname of server + * @param $backupdir directory to operate in + * @param $keep_days number of days to keep backups for + * @param $keep_num number of backups to keep + */ + public function retention( + $internal_username, + $username, + $password, + $url, + $backupdir, + $keep_days, + $keep_num + ) { + // Are we configured to run at all? + // Short circuit in case none of our options are set + if ((strlen($keep_days)) and (strlen($keep_num)) { + return; + } + // Get list of filenames (without path) on remote location + $remote_files = array(); + $tmp_remote_files = $this->listfiles($url, $username, $password, $internal_username, "/$backupdir/", false); + foreach ($tmp_remote_files as $tmp_remote_file) { + if (!($tmp_remote_file === "")) { + if (!($tmp_remote_file == "/$backupdir/")) { + // No idea why the root directory is in the list.. + $remote_files[] = pathinfo($tmp_remote_file)['basename']; + } + } + } + $num_remote_files = count($remote_files); + // Short-circuit, if too few files, no need to check no more + if (!($keep_num === "")) { + if ($keep_num > $num_remote_files) { + return; + } + } + + $date = new \DateTime(); + $files = $this->get_filelist_dates($remote_files); + if (strlen($keep_days)) { + // Admin has specified number of days to keep + $dateinterval = \DateInterval::createFromDateString($keep_days . " day"); + $target_timestamp = date_sub($date, $dateinterval)->format('U'); + // $files is an associative array with key=creation_time, value=filename + // should be sorted by ksort, hopefully that is a numerical sort:) + $new_files = array(); + $old_files = array(); + foreach(array_keys($files) as $file_timestamp) { + if ($file_timestamp > $target_timestamp) { + $new_files[(string)$file_timestamp] = $files[$file_timestamp]; + } else { + // file is "old", aka ripe for deletion + $old_files[(string)$file_timestamp] = $files[$file_timestamp]; + } + } + if (strlen($keep_num)) { + $num_new_files = count($new_files); + if ($num_new_files < $keep_num) { + // Not enough new files to satisfy $keep_num + $missing_num = $keep_num - $num_new_files; + // Can we slice some files from the $old_files list to satisfy $keep_num? + $total_files = count($files); + if ($total_files >= $keep_num) { + // Yes, we can + $tmp_files = array_slice($old_files, 0, $missing_num * -1); + foreach(array_keys($tmp_files) as $filetodelete) { + $this->delete_file($url, $username, $password, $internal_username, $backupdir, $tmp_files[$filetodelete]); + } + } + // No, we can't. Keep all things as is + } else { + // We have more new files than what we need to satisfy $keep_num + foreach(array_keys($old_files) as $filetodelete) { + $this->delete_file($url, $username, $password, $internal_username, $backupdir, $old_files[$filetodelete]); + } + // We do not delete files from new_files, as they are covered by the $keep_days + } + } else { + // We have not been told to keep N items, + // delete everything in $old_files + foreach(array_keys($old_files) as $filetodelete) { + $this->delete_file($url, $username, $password, $internal_username, $backupdir, $old_files[$filetodelete]); + } + } + } else { + // No $keep_days specified + if (strlen($keep_num)) { + // keep_num is some number + // Delete filenames based on their creation time + if (count($files) > $keep_num) { + $tmp_files = array_slice($files, 0, $keep_num * -1); + foreach(array_keys($tmp_files) as $filetodelete) { + $this->delete_file($url, $username, $password, $internal_username, $backupdir, $tmp_files[$filetodelete]); + } + } + } + } + } /** * perform backup @@ -177,6 +451,7 @@ class Nextcloud extends Base implements IBackupProvider */ public function backup() { + date_default_timezone_set('UTC'); $cnf = Config::getInstance(); $nextcloud = new NextcloudSettings(); if ($cnf->isValid() && !empty((string)$nextcloud->enabled)) { @@ -189,6 +464,8 @@ class Nextcloud extends Base implements IBackupProvider $strategy = (string)$nextcloud->strategy; // Strategy 0 = Sync /conf/backup // Strategy 1 = Copy /conf/config.xml to $backupdir/conf-YYYYMMDD.xml + $keep_days = (string)$nextcloud->numdays; + $keep_num = (string)$nextcloud->numbackups; if (!$nextcloud->addhostname->isEmpty()) { $backupdir .= "/".gethostname()."/"; @@ -202,92 +479,13 @@ class Nextcloud extends Base implements IBackupProvider return array(); } - // Backup strategy 1, sync /conf/config.xml to $backupdir/config-YYYYMMDD.xml if ($strategy) { - $confdata = file_get_contents('/conf/config.xml'); - $mdate = filemtime('/conf/config.xml'); - $datestring = date('Ymd', $mdate); - $target_filename = 'config-' . $datestring . '.xml'; - // Find the same filename @ remote - $remote_filename = $url . '/remote.php/dav/files/' . $internal_username . '/' . $backupdir . '/' . $target_filename; - $remote_file_date = $this->get_remote_file_lastmodified($remote_filename, $username, $password); - if ($remote_file_date >= $mdate) { - return array(); - } - // Optionally encrypt - if (!empty($crypto_password)) { - $confdata = $this->encrypt($confdata, $crypto_password); - } - // Finally, upload some data - try { - $this->upload_file_content( - $url, - $username, - $password, - $internal_username, - $backupdir, - $target_filename, - $confdata - ); - return array($backupdir . '/' . $target_filename); - } catch (\Exception $e) { - syslog(LOG_ERR, 'Backup to ' . $url . ' failed: ' . $e); - return array(); - } + $list_of_files = $this->backupstrat_one($internal_username, $username, $password, $url, $backupdir, $crypto_password); + } else { + $list_of_files = $this->backupstrat_zero($internal_username, $username, $password, $url, $backupdir, $crypto_password); } - - // Default strategy (0), sync /conf/backup/ - - // Get list of files from local backup system - $local_files = array(); - $tmp_local_files = scandir('/conf/backup/'); - // Remove '.' and '..', skip directories - foreach ($tmp_local_files as $tmp_local_file) { - if ($tmp_local_file === '.' || $tmp_local_file === '..') { - continue; - } - if (!is_file("/conf/backup/".$tmp_local_file)) { - continue; - } - $local_files[] = $tmp_local_file; - } - - // Get list of filenames (without path) on remote location - $remote_files = array(); - $tmp_remote_files = $this->listfiles($url, $username, $password, $internal_username, "/$backupdir/", false); - foreach ($tmp_remote_files as $tmp_remote_file) { - $remote_files[] = pathinfo($tmp_remote_file)['basename']; - } - - - $uploaded_files = array(); - - // Loop over each local file, - // see if it's in $remote_files, - // if not, optionally encrypt, and upload - foreach ($local_files as $file_to_upload) { - if (!in_array($file_to_upload, $remote_files)) { - $confdata = file_get_contents("/conf/backup/$file_to_upload"); - if (!empty($crypto_password)) { - $confdata = $this->encrypt($confdata, $crypto_password); - } - try { - $this->upload_file_content( - $url, - $username, - $password, - $internal_username, - $backupdir, - $file_to_upload, - $confdata - ); - $uploaded_files[] = $file_to_upload; - } catch (\Exception $e) { - return $uploaded_files; - } - } - } - return $uploaded_files; + $this->retention($internal_username, $username, $password, $url, $backupdir, $keep_days, $keep_num); + return $list_of_files; } } @@ -338,6 +536,7 @@ class Nextcloud extends Base implements IBackupProvider * @param string $url remote location * @param string $username remote user * @param string $password password to use + * @param string $internal_username UUID * @param string $backupdir remote directory * @param string $filename filename to use * @param string $local_file_content contents to save @@ -346,7 +545,7 @@ class Nextcloud extends Base implements IBackupProvider public function upload_file_content($url, $username, $password, $internal_username, $backupdir, $filename, $local_file_content) { $url = $url . "/remote.php/dav/files/$internal_username/$backupdir/$filename"; - $reply = $this->curl_request( + $reply = $this->curl_request_nothrow( $url, $username, $password, @@ -362,6 +561,28 @@ class Nextcloud extends Base implements IBackupProvider } } + /** + * delete a file + * @param string $url remote location + * @param string $username remote user + * @param string $password password to use + * @param strign $internal_username UUID + * @param string $backupdir remote directory + * @param string $filename filename to use + */ + public function delete_file($url, $username, $password, $internal_username, $backupdir, $filename) { + $url = $url . "/remote.php/dav/files/$internal_username/$backupdir/$filename"; + $reply = $this->curl_request_nothrow( + $url, + $username, + $password, + 'DELETE', + 'cannot delete file' + ); + $http_code = $reply['info']['http_code']; + syslog(LOG_ERR, "Deleting " . $url . " returned " . $http_code); + } + /** * create new remote directory if doesn't exist * @param string $url remote location diff --git a/sysutils/nextcloud-backup/src/opnsense/mvc/app/models/OPNsense/Backup/NextcloudSettings.xml b/sysutils/nextcloud-backup/src/opnsense/mvc/app/models/OPNsense/Backup/NextcloudSettings.xml index e18c69d5c..a2599ef5c 100644 --- a/sysutils/nextcloud-backup/src/opnsense/mvc/app/models/OPNsense/Backup/NextcloudSettings.xml +++ b/sysutils/nextcloud-backup/src/opnsense/mvc/app/models/OPNsense/Backup/NextcloudSettings.xml @@ -57,5 +57,13 @@ 1 Y + + 1 + N + + + 1 + N +