PawletOS/android_bootable_recovery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
c45181db0902762fb670b56a173db7c331e564cb
android_bootable_recovery/twrpTar.cpp
Ethan Yonker 1b7a31bd65 Track backup and restore progress 
Track backup and restore progress based on the sizes of the files
as they are being added to the tar backup file. Update the
progress bar based on the sizes of the files.

Change-Id: Idf649efa1db3e91830b4b2add86203a3f30042ff
2014-07-09 08:52:18 -05:00

1393 lines
40 KiB
C++
Raw Blame History

/*
Copyright 2013 TeamWin
This file is part of TWRP/TeamWin Recovery Project.
TWRP is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
TWRP is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with TWRP. If not, see <http://www.gnu.org/licenses/>.
*/
extern "C" {
#include "libtar/libtar.h"
#include "twrpTar.h"
#include "tarWrite.h"
}
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <string.h>
#include <errno.h>
#include <fcntl.h>
#include <fstream>
#include <iostream>
#include <string>
#include <sstream>
#include <vector>
#include <dirent.h>
#include <libgen.h>
#include <sys/mman.h>
#include "twrpTar.hpp"
#include "twcommon.h"
#include "variables.h"
#include "twrp-functions.hpp"
#ifndef BUILD_TWRPTAR_MAIN
#include "data.hpp"
#include "infomanager.hpp"
#endif //ndef BUILD_TWRPTAR_MAIN
using namespace std;
twrpTar::twrpTar(void) {
use_encryption = 0;
userdata_encryption = 0;
use_compression = 0;
split_archives = 0;
has_data_media = 0;
pigz_pid = 0;
oaes_pid = 0;
Total_Backup_Size = 0;
include_root_dir = true;
}
twrpTar::~twrpTar(void) {
// Do nothing
}
void twrpTar::setfn(string fn) {
tarfn = fn;
}
void twrpTar::setdir(string dir) {
tardir = dir;
}
void twrpTar::setsize(unsigned long long backup_size) {
Total_Backup_Size = backup_size;
}
void twrpTar::setpassword(string pass) {
password = pass;
}
int twrpTar::createTarFork(const unsigned long long *overall_size, const unsigned long long *other_backups_size) {
int status = 0;
pid_t pid, rc_pid;
int progress_pipe[2], ret;
file_count = 0;
if (pipe(progress_pipe) < 0) {
LOGERR("Error creating progress tracking pipe\n");
return -1;
}
if ((pid = fork()) == -1) {
LOGINFO("create tar failed to fork.\n");
close(progress_pipe[0]);
close(progress_pipe[1]);
return -1;
}
if (pid == 0) {
// Child process
// Child closes input side of progress pipe
close(progress_pipe[0]);
progress_pipe_fd = progress_pipe[1];
if (use_encryption || userdata_encryption) {
LOGINFO("Using encryption\n");
DIR* d;
struct dirent* de;
unsigned long long regular_size = 0, encrypt_size = 0, target_size = 0, core_count = 1, total_size;
unsigned enc_thread_id = 1, regular_thread_id = 0, i, start_thread_id = 1;
int item_len, ret, thread_error = 0;
std::vector<TarListStruct> RegularList;
std::vector<TarListStruct> EncryptList;
string FileName;
struct TarListStruct TarItem;
twrpTar reg, enc[9];
struct stat st;
pthread_t enc_thread[9];
pthread_attr_t tattr;
void *thread_return;
core_count = sysconf(_SC_NPROCESSORS_CONF);
if (core_count > 8)
core_count = 8;
LOGINFO(" Core Count : %llu\n", core_count);
Archive_Current_Size = 0;
d = opendir(tardir.c_str());
if (d == NULL) {
LOGERR("error opening '%s'\n", tardir.c_str());
close(progress_pipe[1]);
_exit(-1);
}
// Figure out the size of all data to be encrypted and create a list of unencrypted files
while ((de = readdir(d)) != NULL) {
FileName = tardir + "/" + de->d_name;
if (de->d_type == DT_BLK || de->d_type == DT_CHR || du.check_skip_dirs(FileName))
continue;
if (de->d_type == DT_DIR) {
item_len = strlen(de->d_name);
if (userdata_encryption && ((item_len >= 3 && strncmp(de->d_name, "app", 3) == 0) || (item_len >= 6 && strncmp(de->d_name, "dalvik", 6) == 0))) {
ret = Generate_TarList(FileName, &RegularList, &target_size, &regular_thread_id);
if (ret < 0) {
LOGERR("Error in Generate_TarList with regular list!\n");
closedir(d);
close(progress_pipe_fd);
close(progress_pipe[1]);
_exit(-1);
}
file_count = (unsigned long long)(ret);
regular_size += du.Get_Folder_Size(FileName);
} else {
encrypt_size += du.Get_Folder_Size(FileName);
}
} else if (de->d_type == DT_REG) {
stat(FileName.c_str(), &st);
encrypt_size += (unsigned long long)(st.st_size);
}
}
closedir(d);
target_size = encrypt_size / core_count;
target_size++;
LOGINFO(" Unencrypted size: %llu\n", regular_size);
LOGINFO(" Encrypted size : %llu\n", encrypt_size);
LOGINFO(" Target size : %llu\n", target_size);
if (!userdata_encryption) {
enc_thread_id = 0;
start_thread_id = 0;
core_count--;
}
Archive_Current_Size = 0;
d = opendir(tardir.c_str());
if (d == NULL) {
LOGERR("error opening '%s'\n", tardir.c_str());
close(progress_pipe[1]);
_exit(-1);
}
// Divide up the encrypted file list for threading
while ((de = readdir(d)) != NULL) {
FileName = tardir + "/" + de->d_name;
if (de->d_type == DT_BLK || de->d_type == DT_CHR || du.check_skip_dirs(FileName))
continue;
if (de->d_type == DT_DIR) {
item_len = strlen(de->d_name);
if (userdata_encryption && ((item_len >= 3 && strncmp(de->d_name, "app", 3) == 0) || (item_len >= 6 && strncmp(de->d_name, "dalvik", 6) == 0))) {
// Do nothing, we added these to RegularList earlier
} else {
FileName = tardir + "/" + de->d_name;
ret = Generate_TarList(FileName, &EncryptList, &target_size, &enc_thread_id);
if (ret < 0) {
LOGERR("Error in Generate_TarList with encrypted list!\n");
closedir(d);
close(progress_pipe[1]);
_exit(-1);
}
file_count += (unsigned long long)(ret);
}
} else if (de->d_type == DT_REG || de->d_type == DT_LNK) {
stat(FileName.c_str(), &st);
if (de->d_type == DT_REG)
Archive_Current_Size += (unsigned long long)(st.st_size);
TarItem.fn = FileName;
TarItem.thread_id = enc_thread_id;
EncryptList.push_back(TarItem);
file_count++;
}
}
closedir(d);
if (enc_thread_id != core_count) {
LOGERR("Error dividing up threads for encryption, %i threads for %i cores!\n", enc_thread_id, core_count);
if (enc_thread_id > core_count) {
close(progress_pipe[1]);
_exit(-1);
} else {
LOGERR("Continuining anyway.");
}
}
// Send file count to parent
write(progress_pipe_fd, &file_count, sizeof(file_count));
// Send backup size to parent
total_size = regular_size + encrypt_size;
write(progress_pipe_fd, &total_size, sizeof(total_size));
if (userdata_encryption) {
// Create a backup of unencrypted data
reg.setfn(tarfn);
reg.ItemList = &RegularList;
reg.thread_id = 0;
reg.use_encryption = 0;
reg.use_compression = use_compression;
reg.split_archives = 1;
reg.progress_pipe_fd = progress_pipe_fd;
LOGINFO("Creating unencrypted backup...\n");
if (createList((void*)&reg) != 0) {
LOGERR("Error creating unencrypted backup.\n");
close(progress_pipe[1]);
_exit(-1);
}
}
if (pthread_attr_init(&tattr)) {
LOGERR("Unable to pthread_attr_init\n");
close(progress_pipe[1]);
_exit(-1);
}
if (pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_JOINABLE)) {
LOGERR("Error setting pthread_attr_setdetachstate\n");
close(progress_pipe[1]);
_exit(-1);
}
if (pthread_attr_setscope(&tattr, PTHREAD_SCOPE_SYSTEM)) {
LOGERR("Error setting pthread_attr_setscope\n");
close(progress_pipe[1]);
_exit(-1);
}
/*if (pthread_attr_setstacksize(&tattr, 524288)) {
LOGERR("Error setting pthread_attr_setstacksize\n");
_exit(-1);
}*/
// Create threads for the divided up encryption lists
for (i = start_thread_id; i <= core_count; i++) {
enc[i].setdir(tardir);
enc[i].setfn(tarfn);
enc[i].ItemList = &EncryptList;
enc[i].thread_id = i;
enc[i].use_encryption = use_encryption;
enc[i].setpassword(password);
enc[i].use_compression = use_compression;
enc[i].split_archives = 1;
enc[i].progress_pipe_fd = progress_pipe_fd;
LOGINFO("Start encryption thread %i\n", i);
ret = pthread_create(&enc_thread[i], &tattr, createList, (void*)&enc[i]);
if (ret) {
LOGINFO("Unable to create %i thread for encryption! %i\nContinuing in same thread (backup will be slower).\n", i, ret);
if (createList((void*)&enc[i]) != 0) {
LOGERR("Error creating encrypted backup %i.\n", i);
close(progress_pipe[1]);
_exit(-1);
} else {
enc[i].thread_id = i + 1;
}
}
usleep(100000); // Need a short delay before starting the next thread or the threads will never finish for some reason.
}
if (pthread_attr_destroy(&tattr)) {
LOGERR("Failed to pthread_attr_destroy\n");
}
for (i = start_thread_id; i <= core_count; i++) {
if (enc[i].thread_id == i) {
if (pthread_join(enc_thread[i], &thread_return)) {
LOGERR("Error joining thread %i\n", i);
close(progress_pipe[1]);
_exit(-1);
} else {
LOGINFO("Joined thread %i.\n", i);
ret = (int)thread_return;
if (ret != 0) {
thread_error = 1;
LOGERR("Thread %i returned an error %i.\n", i, ret);
close(progress_pipe[1]);
_exit(-1);
}
}
} else {
LOGINFO("Skipping joining thread %i because of pthread failure.\n", i);
}
}
if (thread_error) {
LOGERR("Error returned by one or more threads.\n");
close(progress_pipe[1]);
_exit(-1);
}
LOGINFO("Finished encrypted backup.\n");
close(progress_pipe[1]);
_exit(0);
} else {
// Not encrypted
std::vector<TarListStruct> FileList;
unsigned thread_id = 0;
unsigned long long target_size = 0;
twrpTar reg;
int ret;
// Generate list of files to back up
ret = Generate_TarList(tardir, &FileList, &target_size, &thread_id);
if (ret < 0) {
LOGERR("Error in Generate_TarList!\n");
close(progress_pipe[1]);
_exit(-1);
}
file_count = (unsigned long long)(ret);
// Create a backup
reg.setfn(tarfn);
reg.ItemList = &FileList;
reg.thread_id = 0;
reg.use_encryption = 0;
reg.use_compression = use_compression;
reg.setsize(Total_Backup_Size);
reg.progress_pipe_fd = progress_pipe_fd;
if (Total_Backup_Size > MAX_ARCHIVE_SIZE) {
gui_print("Breaking backup file into multiple archives...\n");
reg.split_archives = 1;
} else {
reg.split_archives = 0;
}
LOGINFO("Creating backup...\n");
write(progress_pipe_fd, &file_count, sizeof(file_count));
write(progress_pipe_fd, &Total_Backup_Size, sizeof(Total_Backup_Size));
if (createList((void*)&reg) != 0) {
LOGERR("Error creating backup.\n");
close(progress_pipe[1]);
_exit(-1);
}
close(progress_pipe[1]);
_exit(0);
}
} else {
// Parent side
unsigned long long fs, size_backup, files_backup, total_backup_size;
int first_data = 0;
double display_percent, progress_percent;
char file_progress[1024];
char size_progress[1024];
files_backup = 0;
size_backup = 0;
// Parent closes output side
close(progress_pipe[1]);
// Read progress data from children
while (read(progress_pipe[0], &fs, sizeof(fs)) > 0) {
if (first_data == 0) {
// First incoming data is the file count
file_count = fs;
if (file_count == 0) file_count = 1; // prevent division by 0 below
first_data = 1;
} else if (first_data == 1) {
// Second incoming data is total size
total_backup_size = fs;
first_data = 2;
} else {
files_backup++;
size_backup += fs;
display_percent = (double)(files_backup) / (double)(file_count) * 100;
sprintf(file_progress, "%llu of %llu files, %i%%", files_backup, file_count, (int)(display_percent));
#ifndef BUILD_TWRPTAR_MAIN
DataManager::SetValue("tw_file_progress", file_progress);
display_percent = (double)(size_backup + *other_backups_size) / (double)(*overall_size) * 100;
sprintf(size_progress, "%lluMB of %lluMB, %i%%", (size_backup + *other_backups_size) / 1048576, *overall_size / 1048576, (int)(display_percent));
DataManager::SetValue("tw_size_progress", size_progress);
progress_percent = (display_percent / 100);
DataManager::SetProgress((float)(progress_percent));
#endif //ndef BUILD_TWRPTAR_MAIN
}
}
close(progress_pipe[0]);
#ifndef BUILD_TWRPTAR_MAIN
DataManager::SetValue("tw_file_progress", "");
DataManager::SetValue("tw_size_progress", "");
InfoManager backup_info(backup_folder + partition_name + ".info");
backup_info.SetValue("backup_size", size_backup);
if (use_compression && use_encryption)
backup_info.SetValue("backup_type", 3);
else if (use_encryption)
backup_info.SetValue("backup_type", 2);
else if (use_compression)
backup_info.SetValue("backup_type", 1);
else
backup_info.SetValue("backup_type", 0);
backup_info.SetValue("file_count", files_backup);
backup_info.SaveValues();
#endif //ndef BUILD_TWRPTAR_MAIN
if (TWFunc::Wait_For_Child(pid, &status, "createTarFork()") != 0)
return -1;
}
return 0;
}
int twrpTar::extractTarFork(const unsigned long long *overall_size, unsigned long long *other_backups_size) {
int status = 0;
pid_t pid, rc_pid;
int progress_pipe[2], ret;
if (pipe(progress_pipe) < 0) {
LOGERR("Error creating progress tracking pipe\n");
return -1;
}
pid = fork();
if (pid >= 0) // fork was successful
{
if (pid == 0) // child process
{
close(progress_pipe[0]);
progress_pipe_fd = progress_pipe[1];
if (TWFunc::Path_Exists(tarfn)) {
LOGINFO("Single archive\n");
if (extract() != 0)
_exit(-1);
else
_exit(0);
} else {
LOGINFO("Multiple archives\n");
string temp;
char actual_filename[255];
twrpTar tars[9];
pthread_t tar_thread[9];
pthread_attr_t tattr;
int thread_count = 0, i, start_thread_id = 1, ret, thread_error = 0;
void *thread_return;
basefn = tarfn;
temp = basefn + "%i%02i";
tarfn += "000";
if (!TWFunc::Path_Exists(tarfn)) {
LOGERR("Unable to locate '%s' or '%s'\n", basefn.c_str(), tarfn.c_str());
close(progress_pipe_fd);
_exit(-1);
}
if (TWFunc::Get_File_Type(tarfn) != 2) {
LOGINFO("First tar file '%s' not encrypted\n", tarfn.c_str());
tars[0].basefn = basefn;
tars[0].thread_id = 0;
tars[0].progress_pipe_fd = progress_pipe_fd;
if (extractMulti((void*)&tars[0]) != 0) {
LOGERR("Error extracting split archive.\n");
close(progress_pipe_fd);
_exit(-1);
}
} else {
start_thread_id = 0;
}
// Start threading encrypted restores
if (pthread_attr_init(&tattr)) {
LOGERR("Unable to pthread_attr_init\n");
close(progress_pipe_fd);
_exit(-1);
}
if (pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_JOINABLE)) {
LOGERR("Error setting pthread_attr_setdetachstate\n");
close(progress_pipe_fd);
_exit(-1);
}
if (pthread_attr_setscope(&tattr, PTHREAD_SCOPE_SYSTEM)) {
LOGERR("Error setting pthread_attr_setscope\n");
close(progress_pipe_fd);
_exit(-1);
}
/*if (pthread_attr_setstacksize(&tattr, 524288)) {
LOGERR("Error setting pthread_attr_setstacksize\n");
close(progress_pipe_fd);
_exit(-1);
}*/
for (i = start_thread_id; i < 9; i++) {
sprintf(actual_filename, temp.c_str(), i, 0);
if (TWFunc::Path_Exists(actual_filename)) {
thread_count++;
tars[i].basefn = basefn;
tars[i].setpassword(password);
tars[i].thread_id = i;
tars[i].progress_pipe_fd = progress_pipe_fd;
LOGINFO("Creating extract thread ID %i\n", i);
ret = pthread_create(&tar_thread[i], &tattr, extractMulti, (void*)&tars[i]);
if (ret) {
LOGINFO("Unable to create %i thread for extraction! %i\nContinuing in same thread (restore will be slower).\n", i, ret);
if (extractMulti((void*)&tars[i]) != 0) {
LOGERR("Error extracting backup in thread %i.\n", i);
close(progress_pipe_fd);
_exit(-1);
} else {
tars[i].thread_id = i + 1;
}
}
usleep(100000); // Need a short delay before starting the next thread or the threads will never finish for some reason.
} else {
break;
}
}
for (i = start_thread_id; i < thread_count + start_thread_id; i++) {
if (tars[i].thread_id == i) {
if (pthread_join(tar_thread[i], &thread_return)) {
LOGERR("Error joining thread %i\n", i);
close(progress_pipe_fd);
_exit(-1);
} else {
LOGINFO("Joined thread %i.\n", i);
ret = (int)thread_return;
if (ret != 0) {
thread_error = 1;
LOGERR("Thread %i returned an error %i.\n", i, ret);
close(progress_pipe_fd);
_exit(-1);
}
}
} else {
LOGINFO("Skipping joining thread %i because of pthread failure.\n", i);
}
}
if (thread_error) {
LOGERR("Error returned by one or more threads.\n");
close(progress_pipe_fd);
_exit(-1);
}
LOGINFO("Finished encrypted restore.\n");
close(progress_pipe_fd);
_exit(0);
}
}
else // parent process
{
unsigned long long fs, size_backup;
double display_percent, progress_percent;
char size_progress[1024];
size_backup = 0;
// Parent closes output side
close(progress_pipe[1]);
// Read progress data from children
while (read(progress_pipe[0], &fs, sizeof(fs)) > 0) {
size_backup += fs;
display_percent = (double)(size_backup + *other_backups_size) / (double)(*overall_size) * 100;
sprintf(size_progress, "%lluMB of %lluMB, %i%%", (size_backup + *other_backups_size) / 1048576, *overall_size / 1048576, (int)(display_percent));
progress_percent = (display_percent / 100);
#ifndef BUILD_TWRPTAR_MAIN
DataManager::SetValue("tw_size_progress", size_progress);
DataManager::SetProgress((float)(progress_percent));
#endif //ndef BUILD_TWRPTAR_MAIN
}
close(progress_pipe[0]);
#ifndef BUILD_TWRPTAR_MAIN
DataManager::SetValue("tw_file_progress", "");
#endif //ndef BUILD_TWRPTAR_MAIN
*other_backups_size += size_backup;
if (TWFunc::Wait_For_Child(pid, &status, "extractTarFork()") != 0)
return -1;
}
}
else // fork has failed
{
close(progress_pipe[0]);
close(progress_pipe[1]);
LOGINFO("extract tar failed to fork.\n");
return -1;
}
return 0;
}
int twrpTar::Generate_TarList(string Path, std::vector<TarListStruct> *TarList, unsigned long long *Target_Size, unsigned *thread_id) {
DIR* d;
struct dirent* de;
struct stat st;
string FileName;
struct TarListStruct TarItem;
string::size_type i;
int ret, file_count;
file_count = 0;
d = opendir(Path.c_str());
if (d == NULL) {
LOGERR("Error opening '%s' -- error: %s\n", Path.c_str(), strerror(errno));
closedir(d);
return -1;
}
while ((de = readdir(d)) != NULL) {
FileName = Path + "/" + de->d_name;
if (de->d_type == DT_BLK || de->d_type == DT_CHR || du.check_skip_dirs(FileName))
continue;
TarItem.fn = FileName;
TarItem.thread_id = *thread_id;
if (de->d_type == DT_DIR) {
TarList->push_back(TarItem);
ret = Generate_TarList(FileName, TarList, Target_Size, thread_id);
if (ret < 0)
return -1;
file_count += ret;
} else if (de->d_type == DT_REG || de->d_type == DT_LNK) {
stat(FileName.c_str(), &st);
TarList->push_back(TarItem);
if (de->d_type == DT_REG) {
file_count++;
Archive_Current_Size += st.st_size;
}
if (Archive_Current_Size != 0 && *Target_Size != 0 && Archive_Current_Size > *Target_Size) {
*thread_id = *thread_id + 1;
Archive_Current_Size = 0;
}
}
}
closedir(d);
return file_count;
}
int twrpTar::extractTar() {
char* charRootDir = (char*) tardir.c_str();
if (openTar() == -1)
return -1;
if (tar_extract_all(t, charRootDir, &progress_pipe_fd) != 0) {
LOGERR("Unable to extract tar archive '%s'\n", tarfn.c_str());
return -1;
}
if (tar_close(t) != 0) {
LOGERR("Unable to close tar file\n");
return -1;
}
return 0;
}
int twrpTar::extract() {
Archive_Current_Type = TWFunc::Get_File_Type(tarfn);
if (Archive_Current_Type == 1) {
//if you return the extractTGZ function directly, stack crashes happen
LOGINFO("Extracting gzipped tar\n");
int ret = extractTar();
return ret;
} else if (Archive_Current_Type == 2) {
int ret = TWFunc::Try_Decrypting_File(tarfn, password);
if (ret < 1) {
LOGERR("Failed to decrypt tar file '%s'\n", tarfn.c_str());
return -1;
}
if (ret == 1) {
LOGERR("Decrypted file is not in tar format.\n");
return -1;
}
if (ret == 3) {
LOGINFO("Extracting encrypted and compressed tar.\n");
Archive_Current_Type = 3;
} else
LOGINFO("Extracting encrypted tar.\n");
return extractTar();
} else {
LOGINFO("Extracting uncompressed tar\n");
return extractTar();
}
}
int twrpTar::tarList(std::vector<TarListStruct> *TarList, unsigned thread_id) {
struct stat st;
char buf[PATH_MAX];
int list_size = TarList->size(), i = 0, archive_count = 0;
string temp;
char actual_filename[PATH_MAX];
char *ptr;
unsigned long long fs;
if (split_archives) {
basefn = tarfn;
temp = basefn + "%i%02i";
sprintf(actual_filename, temp.c_str(), thread_id, archive_count);
tarfn = actual_filename;
include_root_dir = true;
} else {
include_root_dir = false;
}
LOGINFO("Creating tar file '%s'\n", tarfn.c_str());
if (createTar() != 0) {
LOGERR("Error creating tar '%s' for thread %i\n", tarfn.c_str(), thread_id);
return -2;
}
Archive_Current_Size = 0;
while (i < list_size) {
if (TarList->at(i).thread_id == thread_id) {
strcpy(buf, TarList->at(i).fn.c_str());
lstat(buf, &st);
if (S_ISREG(st.st_mode)) { // item is a regular file
fs = (unsigned long long)(st.st_size);
if (split_archives && Archive_Current_Size + fs > MAX_ARCHIVE_SIZE) {
if (closeTar() != 0) {
LOGERR("Error closing '%s' on thread %i\n", tarfn.c_str(), thread_id);
return -3;
}
archive_count++;
gui_print("Splitting thread ID %i into archive %i\n", thread_id, archive_count + 1);
if (archive_count > 99) {
LOGERR("Too many archives for thread %i\n", thread_id);
return -4;
}
sprintf(actual_filename, temp.c_str(), thread_id, archive_count);
tarfn = actual_filename;
if (createTar() != 0) {
LOGERR("Error creating tar '%s' for thread %i\n", tarfn.c_str(), thread_id);
return -2;
}
Archive_Current_Size = 0;
}
Archive_Current_Size += fs;
write(progress_pipe_fd, &fs, sizeof(fs));
}
LOGINFO("addFile '%s' including root: %i\n", buf, include_root_dir);
if (addFile(buf, include_root_dir) != 0) {
LOGERR("Error adding file '%s' to '%s'\n", buf, tarfn.c_str());
return -1;
}
}
i++;
}
if (closeTar() != 0) {
LOGERR("Error closing '%s' on thread %i\n", tarfn.c_str(), thread_id);
return -3;
}
LOGINFO("Thread id %i tarList done, %i archives.\n", thread_id, archive_count, i, list_size);
return 0;
}
void* twrpTar::createList(void *cookie) {
twrpTar* threadTar = (twrpTar*) cookie;
if (threadTar->tarList(threadTar->ItemList, threadTar->thread_id) != 0) {
LOGINFO("ERROR tarList for thread ID %i\n", threadTar->thread_id);
return (void*)-2;
}
LOGINFO("Thread ID %i finished successfully.\n", threadTar->thread_id);
return (void*)0;
}
void* twrpTar::extractMulti(void *cookie) {
twrpTar* threadTar = (twrpTar*) cookie;
int archive_count = 0;
string temp = threadTar->basefn + "%i%02i";
char actual_filename[255];
sprintf(actual_filename, temp.c_str(), threadTar->thread_id, archive_count);
while (TWFunc::Path_Exists(actual_filename)) {
threadTar->tarfn = actual_filename;
if (threadTar->extract() != 0) {
LOGINFO("Error extracting '%s' in thread ID %i\n", actual_filename, threadTar->thread_id);
return (void*)-2;
}
archive_count++;
if (archive_count > 99)
break;
sprintf(actual_filename, temp.c_str(), threadTar->thread_id, archive_count);
}
LOGINFO("Thread ID %i finished successfully.\n", threadTar->thread_id);
return (void*)0;
}
int twrpTar::addFilesToExistingTar(vector <string> files, string fn) {
char* charTarFile = (char*) fn.c_str();
if (tar_open(&t, charTarFile, NULL, O_RDONLY | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) == -1)
return -1;
removeEOT(charTarFile);
if (tar_open(&t, charTarFile, NULL, O_WRONLY | O_APPEND | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) == -1)
return -1;
for (unsigned int i = 0; i < files.size(); ++i) {
char* file = (char*) files.at(i).c_str();
if (tar_append_file(t, file, file) == -1)
return -1;
}
if (tar_append_eof(t) == -1)
return -1;
if (tar_close(t) == -1)
return -1;
return 0;
}
int twrpTar::createTar() {
char* charTarFile = (char*) tarfn.c_str();
char* charRootDir = (char*) tardir.c_str();
static tartype_t type = { open, close, read, write_tar };
if (use_encryption && use_compression) {
// Compressed and encrypted
Archive_Current_Type = 3;
LOGINFO("Using encryption and compression...\n");
int i, pipes[4];
if (pipe(pipes) < 0) {
LOGERR("Error creating first pipe\n");
return -1;
}
if (pipe(pipes + 2) < 0) {
LOGERR("Error creating second pipe\n");
return -1;
}
int output_fd = open(tarfn.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
if (output_fd < 0) {
LOGERR("Failed to open '%s'\n", tarfn.c_str());
for (i = 0; i < 4; i++)
close(pipes[i]); // close all
return -1;
}
pigz_pid = fork();
if (pigz_pid < 0) {
LOGERR("pigz fork() failed\n");
close(output_fd);
for (i = 0; i < 4; i++)
close(pipes[i]); // close all
return -1;
} else if (pigz_pid == 0) {
// pigz Child
close(pipes[1]);
close(pipes[2]);
close(0);
dup2(pipes[0], 0);
close(1);
dup2(pipes[3], 1);
if (execlp("pigz", "pigz", "-", NULL) < 0) {
LOGERR("execlp pigz ERROR!\n");
close(output_fd);
close(pipes[0]);
close(pipes[3]);
_exit(-1);
}
} else {
// Parent
oaes_pid = fork();
if (oaes_pid < 0) {
LOGERR("openaes fork() failed\n");
close(output_fd);
for (i = 0; i < 4; i++)
close(pipes[i]); // close all
return -1;
} else if (oaes_pid == 0) {
// openaes Child
close(pipes[0]);
close(pipes[1]);
close(pipes[3]);
close(0);
dup2(pipes[2], 0);
close(1);
dup2(output_fd, 1);
if (execlp("openaes", "openaes", "enc", "--key", password.c_str(), NULL) < 0) {
LOGERR("execlp openaes ERROR!\n");
close(pipes[2]);
close(output_fd);
_exit(-1);
}
} else {
// Parent
close(pipes[0]);
close(pipes[2]);
close(pipes[3]);
fd = pipes[1];
if(tar_fdopen(&t, fd, charRootDir, NULL, O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
close(fd);
LOGERR("tar_fdopen failed\n");
return -1;
}
return 0;
}
}
} else if (use_compression) {
// Compressed
Archive_Current_Type = 1;
LOGINFO("Using compression...\n");
int pigzfd[2];
int output_fd = open(tarfn.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
if (output_fd < 0) {
LOGERR("Failed to open '%s'\n", tarfn.c_str());
close(pigzfd[0]);
return -1;
}
if (pipe(pigzfd) < 0) {
LOGERR("Error creating pipe\n");
close(output_fd);
return -1;
}
pigz_pid = fork();
if (pigz_pid < 0) {
LOGERR("fork() failed\n");
close(output_fd);
close(pigzfd[0]);
close(pigzfd[1]);
return -1;
} else if (pigz_pid == 0) {
// Child
close(pigzfd[1]); // close unused output pipe
dup2(pigzfd[0], 0); // remap stdin
dup2(output_fd, 1); // remap stdout to output file
if (execlp("pigz", "pigz", "-", NULL) < 0) {
LOGERR("execlp pigz ERROR!\n");
close(output_fd);
close(pigzfd[0]);
_exit(-1);
}
} else {
// Parent
close(pigzfd[0]); // close parent input
fd = pigzfd[1]; // copy parent output
if(tar_fdopen(&t, fd, charRootDir, NULL, O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
close(fd);
LOGERR("tar_fdopen failed\n");
return -1;
}
}
} else if (use_encryption) {
// Encrypted
Archive_Current_Type = 2;
LOGINFO("Using encryption...\n");
int oaesfd[2];
int output_fd = open(tarfn.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
if (output_fd < 0) {
LOGERR("Failed to open '%s'\n", tarfn.c_str());
return -1;
}
if (pipe(oaesfd) < 0) {
LOGERR("Error creating pipe\n");
close(output_fd);
return -1;
}
oaes_pid = fork();
if (oaes_pid < 0) {
LOGERR("fork() failed\n");
close(output_fd);
close(oaesfd[0]);
close(oaesfd[1]);
return -1;
} else if (oaes_pid == 0) {
// Child
close(oaesfd[1]); // close unused
dup2(oaesfd[0], 0); // remap stdin
dup2(output_fd, 1); // remap stdout to output file
if (execlp("openaes", "openaes", "enc", "--key", password.c_str(), NULL) < 0) {
LOGERR("execlp openaes ERROR!\n");
close(output_fd);
close(oaesfd[0]);
_exit(-1);
}
} else {
// Parent
close(oaesfd[0]); // close parent input
fd = oaesfd[1]; // copy parent output
if(tar_fdopen(&t, fd, charRootDir, NULL, O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
close(fd);
LOGERR("tar_fdopen failed\n");
return -1;
}
return 0;
}
} else {
// Not compressed or encrypted
init_libtar_buffer(0);
if (tar_open(&t, charTarFile, &type, O_WRONLY | O_CREAT | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) == -1) {
LOGERR("tar_open error opening '%s'\n", tarfn.c_str());
return -1;
}
}
return 0;
}
int twrpTar::openTar() {
char* charRootDir = (char*) tardir.c_str();
char* charTarFile = (char*) tarfn.c_str();
string Password;
if (Archive_Current_Type == 3) {
LOGINFO("Opening encrypted and compressed backup...\n");
int i, pipes[4];
int input_fd = open(tarfn.c_str(), O_RDONLY | O_LARGEFILE);
if (input_fd < 0) {
LOGERR("Failed to open '%s'\n", tarfn.c_str());
return -1;
}
if (pipe(pipes) < 0) {
LOGERR("Error creating first pipe\n");
close(input_fd);
return -1;
}
if (pipe(pipes + 2) < 0) {
LOGERR("Error creating second pipe\n");
close(pipes[0]);
close(pipes[1]);
close(input_fd);
return -1;
}
oaes_pid = fork();
if (oaes_pid < 0) {
LOGERR("pigz fork() failed\n");
close(input_fd);
for (i = 0; i < 4; i++)
close(pipes[i]); // close all
return -1;
} else if (oaes_pid == 0) {
// openaes Child
close(pipes[0]); // Close pipes that are not used by this child
close(pipes[2]);
close(pipes[3]);
close(0);
dup2(input_fd, 0);
close(1);
dup2(pipes[1], 1);
if (execlp("openaes", "openaes", "dec", "--key", password.c_str(), NULL) < 0) {
LOGERR("execlp openaes ERROR!\n");
close(input_fd);
close(pipes[1]);
_exit(-1);
}
} else {
// Parent
pigz_pid = fork();
if (pigz_pid < 0) {
LOGERR("openaes fork() failed\n");
close(input_fd);
for (i = 0; i < 4; i++)
close(pipes[i]); // close all
return -1;
} else if (pigz_pid == 0) {
// pigz Child
close(pipes[1]); // Close pipes not used by this child
close(pipes[2]);
close(0);
dup2(pipes[0], 0);
close(1);
dup2(pipes[3], 1);
if (execlp("pigz", "pigz", "-d", "-c", NULL) < 0) {
LOGERR("execlp pigz ERROR!\n");
close(input_fd);
close(pipes[0]);
close(pipes[3]);
_exit(-1);
}
} else {
// Parent
close(pipes[0]); // Close pipes not used by parent
close(pipes[1]);
close(pipes[3]);
fd = pipes[2];
if(tar_fdopen(&t, fd, charRootDir, NULL, O_RDONLY | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
close(fd);
LOGERR("tar_fdopen failed\n");
return -1;
}
}
}
} else if (Archive_Current_Type == 2) {
LOGINFO("Opening encrypted backup...\n");
int oaesfd[2];
int input_fd = open(tarfn.c_str(), O_RDONLY | O_LARGEFILE);
if (input_fd < 0) {
LOGERR("Failed to open '%s'\n", tarfn.c_str());
return -1;
}
if (pipe(oaesfd) < 0) {
LOGERR("Error creating pipe\n");
close(input_fd);
return -1;
}
oaes_pid = fork();
if (oaes_pid < 0) {
LOGERR("fork() failed\n");
close(input_fd);
close(oaesfd[0]);
close(oaesfd[1]);
return -1;
} else if (oaes_pid == 0) {
// Child
close(oaesfd[0]); // Close unused pipe
close(0); // close stdin
dup2(oaesfd[1], 1); // remap stdout
dup2(input_fd, 0); // remap input fd to stdin
if (execlp("openaes", "openaes", "dec", "--key", password.c_str(), NULL) < 0) {
LOGERR("execlp openaes ERROR!\n");
close(input_fd);
close(oaesfd[1]);
_exit(-1);
}
} else {
// Parent
close(oaesfd[1]); // close parent output
fd = oaesfd[0]; // copy parent input
if(tar_fdopen(&t, fd, charRootDir, NULL, O_RDONLY | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
close(fd);
LOGERR("tar_fdopen failed\n");
return -1;
}
}
} else if (Archive_Current_Type == 1) {
LOGINFO("Opening as a gzip...\n");
int pigzfd[2];
int input_fd = open(tarfn.c_str(), O_RDONLY | O_LARGEFILE);
if (input_fd < 0) {
LOGERR("Failed to open '%s'\n", tarfn.c_str());
return -1;
}
if (pipe(pigzfd) < 0) {
LOGERR("Error creating pipe\n");
close(input_fd);
return -1;
}
pigz_pid = fork();
if (pigz_pid < 0) {
LOGERR("fork() failed\n");
close(input_fd);
close(pigzfd[0]);
close(pigzfd[1]);
return -1;
} else if (pigz_pid == 0) {
// Child
close(pigzfd[0]);
dup2(input_fd, 0); // remap input fd to stdin
dup2(pigzfd[1], 1); // remap stdout
if (execlp("pigz", "pigz", "-d", "-c", NULL) < 0) {
close(pigzfd[1]);
close(input_fd);
LOGERR("execlp openaes ERROR!\n");
_exit(-1);
}
} else {
// Parent
close(pigzfd[1]); // close parent output
fd = pigzfd[0]; // copy parent input
if(tar_fdopen(&t, fd, charRootDir, NULL, O_RDONLY | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
close(fd);
LOGERR("tar_fdopen failed\n");
return -1;
}
}
} else if (tar_open(&t, charTarFile, NULL, O_RDONLY | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
LOGERR("Unable to open tar archive '%s'\n", charTarFile);
return -1;
}
return 0;
}
string twrpTar::Strip_Root_Dir(string Path) {
string temp;
size_t slash;
if (Path.substr(0, 1) == "/")
temp = Path.substr(1, Path.size() - 1);
else
temp = Path;
slash = temp.find("/");
if (slash == string::npos)
return temp;
else {
string stripped;
stripped = temp.substr(slash, temp.size() - slash);
return stripped;
}
return temp;
}
int twrpTar::addFile(string fn, bool include_root) {
char* charTarFile = (char*) fn.c_str();
if (include_root) {
if (tar_append_file(t, charTarFile, NULL) == -1)
return -1;
} else {
string temp = Strip_Root_Dir(fn);
char* charTarPath = (char*) temp.c_str();
if (tar_append_file(t, charTarFile, charTarPath) == -1)
return -1;
}
return 0;
}
int twrpTar::closeTar() {
flush_libtar_buffer(t->fd);
if (tar_append_eof(t) != 0) {
LOGERR("tar_append_eof(): %s\n", strerror(errno));
tar_close(t);
return -1;
}
if (tar_close(t) != 0) {
LOGERR("Unable to close tar archive: '%s'\n", tarfn.c_str());
return -1;
}
if (Archive_Current_Type > 0) {
close(fd);
int status;
if (pigz_pid > 0 && TWFunc::Wait_For_Child(pigz_pid, &status, "pigz") != 0)
return -1;
if (oaes_pid > 0 && TWFunc::Wait_For_Child(oaes_pid, &status, "openaes") != 0)
return -1;
}
free_libtar_buffer();
if (use_compression && !use_encryption) {
string gzname = tarfn + ".gz";
if (TWFunc::Path_Exists(gzname)) {
rename(gzname.c_str(), tarfn.c_str());
}
}
if (TWFunc::Get_File_Size(tarfn) == 0) {
LOGERR("Backup file size for '%s' is 0 bytes.\n", tarfn.c_str());
return -1;
}
return 0;
}
int twrpTar::removeEOT(string tarFile) {
char* charTarFile = (char*) tarFile.c_str();
off_t tarFileEnd;
while (th_read(t) == 0) {
if (TH_ISREG(t))
tar_skip_regfile(t);
tarFileEnd = lseek(t->fd, 0, SEEK_CUR);
}
if (tar_close(t) == -1)
return -1;
if (truncate(charTarFile, tarFileEnd) == -1)
return -1;
return 0;
}
int twrpTar::entryExists(string entry) {
char* searchstr = (char*)entry.c_str();
int ret;
Archive_Current_Type = TWFunc::Get_File_Type(tarfn);
if (openTar() == -1)
ret = 0;
else
ret = tar_find(t, searchstr);
if (closeTar() != 0)
LOGINFO("Unable to close tar after searching for entry.\n");
return ret;
}
unsigned long long twrpTar::get_size() {
if (TWFunc::Path_Exists(tarfn)) {
LOGINFO("Single archive\n");
int type = 0;
return uncompressedSize(tarfn, &type);
} else {
LOGINFO("Multiple archives\n");
string temp;
char actual_filename[255];
int archive_count = 0, i, type = 0, temp_type = 0;
unsigned long long total_restore_size = 0;
basefn = tarfn;
temp = basefn + "%i%02i";
tarfn += "000";
thread_id = 0;
sprintf(actual_filename, temp.c_str(), thread_id, archive_count);
if (!TWFunc::Path_Exists(actual_filename)) {
LOGERR("Unable to locate '%s' or '%s'\n", basefn.c_str(), tarfn.c_str());
return 0;
}
for (i = 0; i < 9; i++) {
archive_count = 0;
sprintf(actual_filename, temp.c_str(), i, archive_count);
while (TWFunc::Path_Exists(actual_filename)) {
total_restore_size += uncompressedSize(actual_filename, &temp_type);
if (temp_type > type)
type = temp_type;
archive_count++;
if (archive_count > 99)
break;
sprintf(actual_filename, temp.c_str(), i, archive_count);
}
}
#ifndef BUILD_TWRPTAR_MAIN
InfoManager backup_info(backup_folder + "/" + partition_name + ".info");
backup_info.SetValue("backup_size", total_restore_size);
backup_info.SetValue("backup_type", type);
backup_info.SaveValues();
#endif //ndef BUILD_TWRPTAR_MAIN
return total_restore_size;
}
return 0;
}
unsigned long long twrpTar::uncompressedSize(string filename, int *archive_type) {
int type = 0;
unsigned long long total_size = 0;
string Tar, Command, result;
vector<string> split;
Tar = TWFunc::Get_Filename(filename);
type = TWFunc::Get_File_Type(filename);
if (type == 0) {
total_size = TWFunc::Get_File_Size(filename);
*archive_type = 0;
} else if (type == 1) {
// Compressed
Command = "pigz -l '" + filename + "'";
/* if we set Command = "pigz -l " + tarfn + " | sed '1d' | cut -f5 -d' '";
we get the uncompressed size at once. */
TWFunc::Exec_Cmd(Command, result);
if (!result.empty()) {
/* Expected output:
compressed original reduced name
95855838 179403776 -1.3% data.yaffs2.win
^
split[5]
*/
split = TWFunc::split_string(result, ' ', true);
if (split.size() > 4)
total_size = atoi(split[5].c_str());
}
*archive_type = 1;
} else if (type == 2) {
// File is encrypted and may be compressed
int ret = TWFunc::Try_Decrypting_File(filename, password);
*archive_type = 2;
if (ret < 1) {
LOGERR("Failed to decrypt tar file '%s'\n", filename.c_str());
total_size = TWFunc::Get_File_Size(filename);
} else if (ret == 1) {
LOGERR("Decrypted file is not in tar format.\n");
total_size = TWFunc::Get_File_Size(filename);
} else if (ret == 3) {
*archive_type = 3;
Command = "openaes dec --key \"" + password + "\" --in '" + filename + "' | pigz -l";
/* if we set Command = "pigz -l " + tarfn + " | sed '1d' | cut -f5 -d' '";
we get the uncompressed size at once. */
TWFunc::Exec_Cmd(Command, result);
if (!result.empty()) {
LOGINFO("result was: '%s'\n", result.c_str());
/* Expected output:
compressed original reduced name
95855838 179403776 -1.3% data.yaffs2.win
^
split[5]
*/
split = TWFunc::split_string(result, ' ', true);
if (split.size() > 4)
total_size = atoi(split[5].c_str());
}
} else {
total_size = TWFunc::Get_File_Size(filename);
}
}
return total_size;
}
extern "C" ssize_t write_tar(int fd, const void *buffer, size_t size) {
return (ssize_t) write_libtar_buffer(fd, buffer, size);
}
Reference in New Issue View Git Blame Copy Permalink