PawletOS/android_bootable_recovery
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
81c29dc7c42cca8dddfcbdeee14ae84ee6090c8b
android_bootable_recovery/twrpTar.cpp
Dees Troy f4766921d4 Ensure root folders get added in split archives 
Make sure that we add the root folder itself to get the perms and
SELinux contexts before splitting to a second archive.

Change-Id: Id93a9f9b4767b61bba38eb0cf7306c81a8540a3c
2013-10-24 21:33:39 +02:00

1364 lines
38 KiB
C++
Raw Blame History

/*
Copyright 2012 bigbiff/Dees_Troy TeamWin
This file is part of TWRP/TeamWin Recovery Project.
TWRP is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
TWRP is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with TWRP. If not, see <http://www.gnu.org/licenses/>.
*/
extern "C" {
#include "libtar/libtar.h"
#include "twrpTar.h"
#include "tarWrite.h"
#include "libcrecovery/common.h"
}
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/wait.h>
#include <string.h>
#include <errno.h>
#include <fcntl.h>
#include <fstream>
#include <iostream>
#include <string>
#include <sstream>
#include <vector>
#include <dirent.h>
#include <libgen.h>
#include <sys/mman.h>
#include "twrpTar.hpp"
#include "twcommon.h"
#include "data.hpp"
#include "variables.h"
#include "twrp-functions.hpp"
using namespace std;
twrpTar::twrpTar(void) {
use_encryption = 0;
userdata_encryption = 0;
use_compression = 0;
split_archives = 0;
has_data_media = 0;
pigz_pid = 0;
oaes_pid = 0;
}
twrpTar::~twrpTar(void) {
// Do nothing
}
void twrpTar::setfn(string fn) {
tarfn = fn;
}
void twrpTar::setdir(string dir) {
tardir = dir;
}
void twrpTar::setexcl(string exclude) {
tarexclude.push_back(exclude);
}
int twrpTar::createTarFork() {
int status = 0;
pid_t pid, rc_pid;
if ((pid = fork()) == -1) {
LOGINFO("create tar failed to fork.\n");
return -1;
}
if (pid == 0) {
// Child process
if (use_encryption || userdata_encryption) {
LOGINFO("Using encryption\n");
DIR* d;
struct dirent* de;
unsigned long long regular_size = 0, encrypt_size = 0, target_size = 0, core_count = 1;
unsigned enc_thread_id = 1, regular_thread_id = 0, i, start_thread_id = 1;
int item_len, ret, thread_error = 0;
std::vector<TarListStruct> RegularList;
std::vector<TarListStruct> EncryptList;
string FileName;
struct TarListStruct TarItem;
twrpTar reg, enc[9];
struct stat st;
pthread_t enc_thread[9];
pthread_attr_t tattr;
void *thread_return;
core_count = sysconf(_SC_NPROCESSORS_CONF);
if (core_count > 8)
core_count = 8;
LOGINFO(" Core Count : %llu\n", core_count);
Archive_Current_Size = 0;
d = opendir(tardir.c_str());
if (d == NULL) {
LOGERR("error opening '%s'\n", tardir.c_str());
_exit(-1);
}
// Figure out the size of all data to be encrypted and create a list of unencrypted files
while ((de = readdir(d)) != NULL) {
FileName = tardir + "/";
FileName += de->d_name;
if (has_data_media == 1 && FileName.size() >= 11 && strncmp(FileName.c_str(), "/data/media", 11) == 0)
continue; // Skip /data/media
if (de->d_type == DT_BLK || de->d_type == DT_CHR)
continue;
if (de->d_type == DT_DIR && strcmp(de->d_name, ".") != 0 && strcmp(de->d_name, "..") != 0 && strcmp(de->d_name, "lost+found") != 0) {
item_len = strlen(de->d_name);
if (userdata_encryption && ((item_len >= 3 && strncmp(de->d_name, "app", 3) == 0) || (item_len >= 6 && strncmp(de->d_name, "dalvik", 6) == 0))) {
if (Generate_TarList(FileName, &RegularList, &target_size, &regular_thread_id) < 0) {
LOGERR("Error in Generate_TarList with regular list!\n");
closedir(d);
_exit(-1);
}
regular_size += TWFunc::Get_Folder_Size(FileName, false);
} else {
encrypt_size += TWFunc::Get_Folder_Size(FileName, false);
}
} else if (de->d_type == DT_REG) {
stat(FileName.c_str(), &st);
encrypt_size += (unsigned long long)(st.st_size);
}
}
closedir(d);
target_size = encrypt_size / core_count;
target_size++;
LOGINFO(" Unencrypted size: %llu\n", regular_size);
LOGINFO(" Encrypted size : %llu\n", encrypt_size);
LOGINFO(" Target size : %llu\n", target_size);
if (!userdata_encryption) {
enc_thread_id = 0;
start_thread_id = 0;
core_count--;
}
Archive_Current_Size = 0;
d = opendir(tardir.c_str());
if (d == NULL) {
LOGERR("error opening '%s'\n", tardir.c_str());
_exit(-1);
}
// Divide up the encrypted file list for threading
while ((de = readdir(d)) != NULL) {
FileName = tardir + "/";
FileName += de->d_name;
if (has_data_media == 1 && FileName.size() >= 11 && strncmp(FileName.c_str(), "/data/media", 11) == 0)
continue; // Skip /data/media
if (de->d_type == DT_BLK || de->d_type == DT_CHR)
continue;
if (de->d_type == DT_DIR && strcmp(de->d_name, ".") != 0 && strcmp(de->d_name, "..") != 0 && strcmp(de->d_name, "lost+found") != 0) {
item_len = strlen(de->d_name);
if (userdata_encryption && ((item_len >= 3 && strncmp(de->d_name, "app", 3) == 0) || (item_len >= 6 && strncmp(de->d_name, "dalvik", 6) == 0))) {
// Do nothing, we added these to RegularList earlier
} else {
FileName = tardir + "/";
FileName += de->d_name;
if (Generate_TarList(FileName, &EncryptList, &target_size, &enc_thread_id) < 0) {
LOGERR("Error in Generate_TarList with encrypted list!\n");
closedir(d);
_exit(-1);
}
}
} else if (de->d_type == DT_REG || de->d_type == DT_LNK) {
stat(FileName.c_str(), &st);
if (de->d_type == DT_REG)
Archive_Current_Size += (unsigned long long)(st.st_size);
TarItem.fn = FileName;
TarItem.thread_id = enc_thread_id;
EncryptList.push_back(TarItem);
}
}
closedir(d);
if (enc_thread_id != core_count) {
LOGERR("Error dividing up threads for encryption, %i threads for %i cores!\n", enc_thread_id, core_count);
if (enc_thread_id > core_count)
_exit(-1);
else
LOGERR("Continuining anyway.");
}
if (userdata_encryption) {
// Create a backup of unencrypted data
reg.setfn(tarfn);
reg.ItemList = &RegularList;
reg.thread_id = 0;
reg.use_encryption = 0;
reg.use_compression = use_compression;
LOGINFO("Creating unencrypted backup...\n");
if (createList((void*)&reg) != 0) {
LOGERR("Error creating unencrypted backup.\n");
_exit(-1);
}
}
if (pthread_attr_init(&tattr)) {
LOGERR("Unable to pthread_attr_init\n");
_exit(-1);
}
if (pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_JOINABLE)) {
LOGERR("Error setting pthread_attr_setdetachstate\n");
_exit(-1);
}
if (pthread_attr_setscope(&tattr, PTHREAD_SCOPE_SYSTEM)) {
LOGERR("Error setting pthread_attr_setscope\n");
_exit(-1);
}
/*if (pthread_attr_setstacksize(&tattr, 524288)) {
LOGERR("Error setting pthread_attr_setstacksize\n");
_exit(-1);
}*/
// Create threads for the divided up encryption lists
for (i = start_thread_id; i <= core_count; i++) {
enc[i].setdir(tardir);
enc[i].setfn(tarfn);
enc[i].ItemList = &EncryptList;
enc[i].thread_id = i;
enc[i].use_encryption = use_encryption;
enc[i].use_compression = use_compression;
LOGINFO("Start encryption thread %i\n", i);
ret = pthread_create(&enc_thread[i], &tattr, createList, (void*)&enc[i]);
if (ret) {
LOGINFO("Unable to create %i thread for encryption! %i\nContinuing in same thread (backup will be slower).", i, ret);
if (createList((void*)&enc[i]) != 0) {
LOGERR("Error creating encrypted backup %i.\n", i);
_exit(-1);
} else {
enc[i].thread_id = i + 1;
}
}
usleep(100000); // Need a short delay before starting the next thread or the threads will never finish for some reason.
}
if (pthread_attr_destroy(&tattr)) {
LOGERR("Failed to pthread_attr_destroy\n");
}
for (i = start_thread_id; i <= core_count; i++) {
if (enc[i].thread_id == i) {
if (pthread_join(enc_thread[i], &thread_return)) {
LOGERR("Error joining thread %i\n", i);
_exit(-1);
} else {
LOGINFO("Joined thread %i.\n", i);
ret = (int)thread_return;
if (ret != 0) {
thread_error = 1;
LOGERR("Thread %i returned an error %i.\n", i, ret);
_exit(-1);
}
}
} else {
LOGINFO("Skipping joining thread %i because of pthread failure.\n", i);
}
}
if (thread_error) {
LOGERR("Error returned by one or more threads.\n");
_exit(-1);
}
LOGINFO("Finished encrypted backup.\n");
_exit(0);
} else {
if (create() != 0)
_exit(-1);
else
_exit(0);
}
} else {
if (TWFunc::Wait_For_Child(pid, &status, "createTarFork()") != 0)
return -1;
}
return 0;
}
int twrpTar::extractTarFork() {
int status = 0;
pid_t pid, rc_pid;
pid = fork();
if (pid >= 0) // fork was successful
{
if (pid == 0) // child process
{
if (TWFunc::Path_Exists(tarfn)) {
LOGINFO("Single archive\n");
if (extract() != 0)
_exit(-1);
else
_exit(0);
} else {
LOGINFO("Multiple archives\n");
string temp;
char actual_filename[255];
twrpTar tars[9];
pthread_t tar_thread[9];
pthread_attr_t tattr;
int thread_count = 0, i, start_thread_id = 1, ret, thread_error = 0;
void *thread_return;
basefn = tarfn;
temp = basefn + "%i%02i";
tarfn += "000";
if (!TWFunc::Path_Exists(tarfn)) {
LOGERR("Unable to locate '%s' or '%s'\n", basefn.c_str(), tarfn.c_str());
_exit(-1);
}
if (TWFunc::Get_File_Type(tarfn) != 2) {
LOGINFO("First tar file '%s' not encrypted\n", tarfn.c_str());
tars[0].basefn = basefn;
tars[0].thread_id = 0;
if (extractMulti((void*)&tars[0]) != 0) {
LOGERR("Error extracting split archive.\n");
_exit(-1);
}
} else {
start_thread_id = 0;
}
// Start threading encrypted restores
if (pthread_attr_init(&tattr)) {
LOGERR("Unable to pthread_attr_init\n");
_exit(-1);
}
if (pthread_attr_setdetachstate(&tattr, PTHREAD_CREATE_JOINABLE)) {
LOGERR("Error setting pthread_attr_setdetachstate\n");
_exit(-1);
}
if (pthread_attr_setscope(&tattr, PTHREAD_SCOPE_SYSTEM)) {
LOGERR("Error setting pthread_attr_setscope\n");
_exit(-1);
}
/*if (pthread_attr_setstacksize(&tattr, 524288)) {
LOGERR("Error setting pthread_attr_setstacksize\n");
_exit(-1);
}*/
for (i = start_thread_id; i < 9; i++) {
sprintf(actual_filename, temp.c_str(), i, 0);
if (TWFunc::Path_Exists(actual_filename)) {
thread_count++;
tars[i].basefn = basefn;
tars[i].thread_id = i;
LOGINFO("Creating extract thread ID %i\n", i);
ret = pthread_create(&tar_thread[i], &tattr, extractMulti, (void*)&tars[i]);
if (ret) {
LOGINFO("Unable to create %i thread for extraction! %i\nContinuing in same thread (restore will be slower).", i, ret);
if (extractMulti((void*)&tars[i]) != 0) {
LOGERR("Error extracting backup in thread %i.\n", i);
_exit(-1);
} else {
tars[i].thread_id = i + 1;
}
}
usleep(100000); // Need a short delay before starting the next thread or the threads will never finish for some reason.
} else {
break;
}
}
for (i = start_thread_id; i < thread_count + start_thread_id; i++) {
if (tars[i].thread_id == i) {
if (pthread_join(tar_thread[i], &thread_return)) {
LOGERR("Error joining thread %i\n", i);
_exit(-1);
} else {
LOGINFO("Joined thread %i.\n", i);
ret = (int)thread_return;
if (ret != 0) {
thread_error = 1;
LOGERR("Thread %i returned an error %i.\n", i, ret);
_exit(-1);
}
}
} else {
LOGINFO("Skipping joining thread %i because of pthread failure.\n", i);
}
}
if (thread_error) {
LOGERR("Error returned by one or more threads.\n");
_exit(-1);
}
LOGINFO("Finished encrypted backup.\n");
_exit(0);
}
}
else // parent process
{
if (TWFunc::Wait_For_Child(pid, &status, "extractTarFork()") != 0)
return -1;
}
}
else // fork has failed
{
LOGINFO("extract tar failed to fork.\n");
return -1;
}
return 0;
}
int twrpTar::splitArchiveFork() {
int status = 0;
pid_t pid, rc_pid;
pid = fork();
if (pid >= 0) // fork was successful
{
if (pid == 0) // child process
{
if (Split_Archive() <= 0)
_exit(-1);
else
_exit(0);
}
else // parent process
{
if (TWFunc::Wait_For_Child(pid, &status, "splitArchiveFork()") != 0)
return -1;
}
}
else // fork has failed
{
LOGINFO("split archive failed to fork.\n");
return -1;
}
return 0;
}
int twrpTar::Generate_TarList(string Path, std::vector<TarListStruct> *TarList, unsigned long long *Target_Size, unsigned *thread_id) {
DIR* d;
struct dirent* de;
struct stat st;
string FileName;
struct TarListStruct TarItem;
string::size_type i;
bool skip;
if (has_data_media == 1 && Path.size() >= 11 && strncmp(Path.c_str(), "/data/media", 11) == 0)
return 0; // Skip /data/media
d = opendir(Path.c_str());
if (d == NULL) {
LOGERR("error opening '%s' -- error: %s\n", Path.c_str(), strerror(errno));
closedir(d);
return -1;
}
while ((de = readdir(d)) != NULL) {
// Skip excluded stuff
if (split.size() > 0) {
skip = false;
for (i = 0; i < split.size(); i++) {
if (strcmp(de->d_name, split[i].c_str()) == 0) {
LOGINFO("excluding %s\n", de->d_name);
skip = true;
break;
}
}
if (skip)
continue;
}
FileName = Path + "/";
FileName += de->d_name;
if (has_data_media == 1 && FileName.size() >= 11 && strncmp(FileName.c_str(), "/data/media", 11) == 0)
continue; // Skip /data/media
if (de->d_type == DT_BLK || de->d_type == DT_CHR)
continue;
TarItem.fn = FileName;
TarItem.thread_id = *thread_id;
if (de->d_type == DT_DIR && strcmp(de->d_name, ".") != 0 && strcmp(de->d_name, "..") != 0 && strcmp(de->d_name, "lost+found") != 0) {
TarList->push_back(TarItem);
if (Generate_TarList(FileName, TarList, Target_Size, thread_id) < 0)
return -1;
} else if (de->d_type == DT_REG || de->d_type == DT_LNK) {
stat(FileName.c_str(), &st);
TarList->push_back(TarItem);
if (de->d_type == DT_REG)
Archive_Current_Size += st.st_size;
if (Archive_Current_Size != 0 && *Target_Size != 0 && Archive_Current_Size > *Target_Size) {
*thread_id = *thread_id + 1;
Archive_Current_Size = 0;
}
}
}
closedir(d);
return 0;
}
int twrpTar::Generate_Multiple_Archives(string Path) {
DIR* d;
struct dirent* de;
struct stat st;
string FileName;
char actual_filename[255];
string::size_type i;
bool skip;
if (has_data_media == 1 && Path.size() >= 11 && strncmp(Path.c_str(), "/data/media", 11) == 0)
return 0; // Skip /data/media
LOGINFO("Path: '%s', archive filename: '%s'\n", Path.c_str(), tarfn.c_str());
d = opendir(Path.c_str());
if (d == NULL)
{
LOGERR("error opening '%s' -- error: %s\n", Path.c_str(), strerror(errno));
closedir(d);
return -1;
}
while ((de = readdir(d)) != NULL) {
// Skip excluded stuff
if (split.size() > 0) {
skip = false;
for (i = 0; i < split.size(); i++) {
if (strcmp(de->d_name, split[i].c_str()) == 0) {
LOGINFO("excluding %s\n", de->d_name);
skip = true;
break;
}
}
if (skip) {
continue;
}
}
FileName = Path + "/";
FileName += de->d_name;
if (has_data_media == 1 && FileName.size() >= 11 && strncmp(FileName.c_str(), "/data/media", 11) == 0)
continue; // Skip /data/media
if (de->d_type == DT_BLK || de->d_type == DT_CHR)
continue;
if (de->d_type == DT_DIR && strcmp(de->d_name, ".") != 0 && strcmp(de->d_name, "..") != 0)
{
unsigned long long folder_size = TWFunc::Get_Folder_Size(FileName, false);
if (Archive_Current_Size + folder_size > MAX_ARCHIVE_SIZE) {
// Add the root folder first
LOGINFO("Adding root folder '%s' before splitting.\n", FileName.c_str());
if (addFile(FileName, true) != 0) {
LOGERR("Error adding folder '%s' to split archive.\n", FileName.c_str());
return -1;
}
LOGINFO("Calling Generate_Multiple_Archives\n");
if (Generate_Multiple_Archives(FileName) < 0)
return -1;
} else {
//FileName += "/";
LOGINFO("Adding folder '%s'\n", FileName.c_str());
tardir = FileName;
if (tarDirs(true) < 0)
return -1;
Archive_Current_Size += folder_size;
}
}
else if (de->d_type == DT_REG || de->d_type == DT_LNK)
{
stat(FileName.c_str(), &st);
if (de->d_type != DT_LNK) {
if (Archive_Current_Size != 0 && Archive_Current_Size + st.st_size > MAX_ARCHIVE_SIZE) {
LOGINFO("Closing tar '%s', ", tarfn.c_str());
closeTar();
if (TWFunc::Get_File_Size(tarfn) == 0) {
LOGERR("Backup file size for '%s' is 0 bytes.\n", tarfn.c_str());
return -1;
}
Archive_File_Count++;
if (Archive_File_Count > 999) {
LOGERR("Archive count is too large!\n");
return -1;
}
string temp = basefn + "%03i";
sprintf(actual_filename, temp.c_str(), Archive_File_Count);
tarfn = actual_filename;
Archive_Current_Size = 0;
LOGINFO("Creating tar '%s'\n", tarfn.c_str());
gui_print("Creating archive %i...\n", Archive_File_Count + 1);
if (createTar() != 0)
return -1;
}
}
LOGINFO("Adding file: '%s'... ", FileName.c_str());
if (addFile(FileName, true) < 0)
return -1;
if (de->d_type != DT_LNK) {
Archive_Current_Size += st.st_size;
}
LOGINFO("added successfully, archive size: %llu\n", Archive_Current_Size);
if (de->d_type != DT_LNK) {
if (st.st_size > 2147483648LL)
LOGERR("There is a file that is larger than 2GB in the file system\n'%s'\nThis file may not restore properly\n", FileName.c_str());
}
}
}
closedir(d);
return 0;
}
int twrpTar::Split_Archive()
{
string temp = tarfn + "%03i";
char actual_filename[255];
string tarsplit;
basefn = tarfn;
Archive_File_Count = 0;
Archive_Current_Size = 0;
sprintf(actual_filename, temp.c_str(), Archive_File_Count);
tarfn = actual_filename;
for (int i = 0; i < tarexclude.size(); ++i) {
tarsplit = tarexclude[i];
tarsplit += " ";
}
if (!tarexclude.empty())
split = TWFunc::split_string(tarsplit, ' ', true);
createTar();
DataManager::GetValue(TW_HAS_DATA_MEDIA, has_data_media);
gui_print("Creating archive 1...\n");
if (Generate_Multiple_Archives(tardir) < 0) {
LOGERR("Error generating multiple archives\n");
return -1;
}
closeTar();
LOGINFO("Done, created %i archives.\n", (++Archive_File_Count));
return (Archive_File_Count);
}
int twrpTar::extractTar() {
char* charRootDir = (char*) tardir.c_str();
if (openTar() == -1)
return -1;
if (tar_extract_all(t, charRootDir) != 0) {
LOGERR("Unable to extract tar archive '%s'\n", tarfn.c_str());
return -1;
}
if (tar_close(t) != 0) {
LOGERR("Unable to close tar file\n");
return -1;
}
return 0;
}
int twrpTar::extract() {
Archive_Current_Type = TWFunc::Get_File_Type(tarfn);
if (Archive_Current_Type == 1) {
//if you return the extractTGZ function directly, stack crashes happen
LOGINFO("Extracting gzipped tar\n");
int ret = extractTar();
return ret;
} else if (Archive_Current_Type == 2) {
string Password;
DataManager::GetValue("tw_restore_password", Password);
int ret = TWFunc::Try_Decrypting_File(tarfn, Password);
if (ret < 1) {
LOGERR("Failed to decrypt tar file '%s'\n", tarfn.c_str());
return -1;
}
if (ret == 1) {
LOGERR("Decrypted file is not in tar format.\n");
return -1;
}
if (ret == 3) {
LOGINFO("Extracting encrypted and compressed tar.\n");
Archive_Current_Type = 3;
} else
LOGINFO("Extracting encrypted tar.\n");
return extractTar();
} else {
LOGINFO("Extracting uncompressed tar\n");
return extractTar();
}
}
int twrpTar::tarDirs(bool include_root) {
DIR* d;
string mainfolder = tardir + "/", subfolder;
string tarsplit;
char buf[PATH_MAX], charTarPath[PATH_MAX];
string excl;
string::size_type i;
bool skip;
//set exclude directories for libtar
for (int i = 0; i < tarexclude.size(); ++i) {
excl += tarexclude.at(i);
tarsplit = tarexclude.at(i);
excl += " ";
tarsplit += " ";
}
d = opendir(tardir.c_str());
if (d != NULL) {
if (!tarsplit.empty()) {
split = TWFunc::split_string(tarsplit, ' ', true);
}
struct dirent* de;
while ((de = readdir(d)) != NULL) {
#ifdef RECOVERY_SDCARD_ON_DATA
if ((tardir == "/data" || tardir == "/data/") && strcmp(de->d_name, "media") == 0) continue;
#endif
if (de->d_type == DT_BLK || de->d_type == DT_CHR || strcmp(de->d_name, "..") == 0 || strcmp(de->d_name, "lost+found") == 0)
continue;
// Skip excluded stuff
if (split.size() > 0) {
skip = false;
for (i = 0; i < split.size(); i++) {
if (strcmp(de->d_name, split[i].c_str()) == 0) {
LOGINFO("excluding %s\n", de->d_name);
skip = true;
break;
}
}
if (skip) {
continue;
}
}
subfolder = mainfolder;
if (strcmp(de->d_name, ".") != 0) {
subfolder += de->d_name;
} else {
std::string parentDir = basename(subfolder.c_str());
LOGINFO("parentDir: %s\n", parentDir.c_str());
if (!parentDir.compare("lost+found"))
continue;
LOGINFO("tarDirs addFile '%s' including root: %i\n", subfolder.c_str(), include_root);
if (addFile(subfolder, include_root) != 0)
return -1;
continue;
}
strcpy(buf, subfolder.c_str());
if (de->d_type == DT_DIR) {
if (include_root) {
charTarPath[0] = NULL;
LOGINFO("tar_append_tree '%s' as NULL\n", buf, charTarPath);
if (tar_append_tree(t, buf, NULL, &excl[0]) != 0) {
LOGERR("Error appending '%s' to tar archive '%s'\n", buf, tarfn.c_str());
return -1;
}
} else {
string temp = Strip_Root_Dir(buf);
strcpy(charTarPath, temp.c_str());
LOGINFO("tar_append_tree '%s' as '%s'\n", buf, charTarPath);
if (tar_append_tree(t, buf, charTarPath, &excl[0]) != 0) {
LOGERR("Error appending '%s' to tar archive '%s'\n", buf, tarfn.c_str());
return -1;
}
}
} else if (tardir != "/" && (de->d_type == DT_REG || de->d_type == DT_LNK)) {
LOGINFO("addFile '%s' including root: %i\n", buf, include_root);
if (addFile(buf, include_root) != 0)
return -1;
}
fflush(NULL);
}
closedir(d);
}
return 0;
}
int twrpTar::tarList(bool include_root, std::vector<TarListStruct> *TarList, unsigned thread_id) {
struct stat st;
char buf[PATH_MAX];
int list_size = TarList->size(), i = 0, archive_count = 0;
string temp;
char actual_filename[PATH_MAX];
basefn = tarfn;
temp = basefn + "%i%02i";
sprintf(actual_filename, temp.c_str(), thread_id, archive_count);
tarfn = actual_filename;
if (createTar() != 0) {
LOGERR("Error creating tar '%s' for thread %i\n", tarfn.c_str(), thread_id);
return -2;
}
Archive_Current_Size = 0;
while (i < list_size) {
if (TarList->at(i).thread_id == thread_id) {
strcpy(buf, TarList->at(i).fn.c_str());
stat(buf, &st);
if (st.st_mode & S_IFREG) { // item is a regular file
if (Archive_Current_Size + (unsigned long long)(st.st_size) > MAX_ARCHIVE_SIZE) {
if (closeTar() != 0) {
LOGERR("Error closing '%s' on thread %i\n", tarfn.c_str(), thread_id);
return -3;
}
archive_count++;
LOGINFO("Splitting thread ID %i into archive %i\n", thread_id, archive_count);
if (archive_count > 99) {
LOGINFO("BLAH!\n");
LOGERR("Too many archives for thread %i\n", thread_id);
return -4;
}
sprintf(actual_filename, temp.c_str(), thread_id, archive_count);
tarfn = actual_filename;
if (createTar() != 0) {
LOGERR("Error creating tar '%s' for thread %i\n", tarfn.c_str(), thread_id);
return -2;
}
Archive_Current_Size = 0;
}
Archive_Current_Size += (unsigned long long)(st.st_size);
}
if (addFile(buf, include_root) != 0) {
LOGERR("Error adding file '%s' to '%s'\n", buf, tarfn.c_str());
return -1;
}
}
i++;
}
if (closeTar() != 0) {
LOGERR("Error closing '%s' on thread %i\n", tarfn.c_str(), thread_id);
return -3;
}
LOGINFO("Thread id %i tarList done, %i archives.\n", thread_id, archive_count, i, list_size);
return 0;
}
int twrpTar::create() {
init_libtar_buffer(0);
if (createTar() == -1)
return -1;
if (tarDirs(false) == -1)
return -1;
if (closeTar() == -1)
return -1;
free_libtar_buffer();
return 0;
}
void* twrpTar::createList(void *cookie) {
twrpTar* threadTar = (twrpTar*) cookie;
if (threadTar->tarList(true, threadTar->ItemList, threadTar->thread_id) == -1) {
LOGINFO("ERROR tarList for thread ID %i\n", threadTar->thread_id);
return (void*)-2;
}
LOGINFO("Thread ID %i finished successfully.\n", threadTar->thread_id);
return (void*)0;
}
void* twrpTar::extractMulti(void *cookie) {
twrpTar* threadTar = (twrpTar*) cookie;
int archive_count = 0;
string temp = threadTar->basefn + "%i%02i";
char actual_filename[255];
sprintf(actual_filename, temp.c_str(), threadTar->thread_id, archive_count);
while (TWFunc::Path_Exists(actual_filename)) {
threadTar->tarfn = actual_filename;
if (threadTar->extract() != 0) {
LOGINFO("Error extracting '%s' in thread ID %i\n", actual_filename, threadTar->thread_id);
return (void*)-2;
}
archive_count++;
if (archive_count > 99)
break;
sprintf(actual_filename, temp.c_str(), threadTar->thread_id, archive_count);
}
LOGINFO("Thread ID %i finished successfully.\n", threadTar->thread_id);
return (void*)0;
}
int twrpTar::addFilesToExistingTar(vector <string> files, string fn) {
char* charTarFile = (char*) fn.c_str();
if (tar_open(&t, charTarFile, NULL, O_RDONLY | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) == -1)
return -1;
removeEOT(charTarFile);
if (tar_open(&t, charTarFile, NULL, O_WRONLY | O_APPEND | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) == -1)
return -1;
for (unsigned int i = 0; i < files.size(); ++i) {
char* file = (char*) files.at(i).c_str();
if (tar_append_file(t, file, file) == -1)
return -1;
}
if (tar_append_eof(t) == -1)
return -1;
if (tar_close(t) == -1)
return -1;
return 0;
}
int twrpTar::createTar() {
char* charTarFile = (char*) tarfn.c_str();
char* charRootDir = (char*) tardir.c_str();
static tartype_t type = { open, close, read, write_tar };
string Password;
if (use_encryption && use_compression) {
// Compressed and encrypted
Archive_Current_Type = 3;
LOGINFO("Using encryption and compression...\n");
DataManager::GetValue("tw_backup_password", Password);
int i, pipes[4];
if (pipe(pipes) < 0) {
LOGERR("Error creating first pipe\n");
return -1;
}
if (pipe(pipes + 2) < 0) {
LOGERR("Error creating second pipe\n");
return -1;
}
pigz_pid = fork();
if (pigz_pid < 0) {
LOGERR("pigz fork() failed\n");
for (i = 0; i < 4; i++)
close(pipes[i]); // close all
return -1;
} else if (pigz_pid == 0) {
// pigz Child
close(pipes[1]);
close(pipes[2]);
close(0);
dup2(pipes[0], 0);
close(1);
dup2(pipes[3], 1);
if (execlp("pigz", "pigz", "-", NULL) < 0) {
LOGERR("execlp pigz ERROR!\n");
close(pipes[0]);
close(pipes[3]);
_exit(-1);
}
} else {
// Parent
oaes_pid = fork();
if (oaes_pid < 0) {
LOGERR("openaes fork() failed\n");
for (i = 0; i < 4; i++)
close(pipes[i]); // close all
return -1;
} else if (oaes_pid == 0) {
// openaes Child
int output_fd = open(tarfn.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
if (output_fd < 0) {
LOGERR("Failed to open '%s'\n", tarfn.c_str());
for (i = 0; i < 4; i++)
close(pipes[i]); // close all
return -1;
}
close(pipes[0]);
close(pipes[1]);
close(pipes[3]);
close(0);
dup2(pipes[2], 0);
close(1);
dup2(output_fd, 1);
if (execlp("openaes", "openaes", "enc", "--key", Password.c_str(), NULL) < 0) {
LOGERR("execlp openaes ERROR!\n");
close(pipes[2]);
close(output_fd);
_exit(-1);
}
} else {
// Parent
close(pipes[0]);
close(pipes[2]);
close(pipes[3]);
fd = pipes[1];
if(tar_fdopen(&t, fd, charRootDir, NULL, O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
close(fd);
LOGERR("tar_fdopen failed\n");
return -1;
}
return 0;
}
}
} else if (use_compression) {
// Compressed
Archive_Current_Type = 1;
LOGINFO("Using compression...\n");
int pigzfd[2];
if (pipe(pigzfd) < 0) {
LOGERR("Error creating pipe\n");
return -1;
}
pigz_pid = fork();
if (pigz_pid < 0) {
LOGERR("fork() failed\n");
close(pigzfd[0]);
close(pigzfd[1]);
return -1;
} else if (pigz_pid == 0) {
// Child
close(pigzfd[1]); // close unused output pipe
int output_fd = open(tarfn.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
if (output_fd < 0) {
LOGERR("Failed to open '%s'\n", tarfn.c_str());
close(pigzfd[0]);
_exit(-1);
}
dup2(pigzfd[0], 0); // remap stdin
dup2(output_fd, 1); // remap stdout to output file
if (execlp("pigz", "pigz", "-", NULL) < 0) {
LOGERR("execlp pigz ERROR!\n");
close(output_fd);
close(pigzfd[0]);
_exit(-1);
}
} else {
// Parent
close(pigzfd[0]); // close parent input
fd = pigzfd[1]; // copy parent output
if(tar_fdopen(&t, fd, charRootDir, NULL, O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
close(fd);
LOGERR("tar_fdopen failed\n");
return -1;
}
}
} else if (use_encryption) {
// Encrypted
Archive_Current_Type = 2;
LOGINFO("Using encryption...\n");
DataManager::GetValue("tw_backup_password", Password);
int oaesfd[2];
pipe(oaesfd);
oaes_pid = fork();
if (oaes_pid < 0) {
LOGERR("fork() failed\n");
close(oaesfd[0]);
close(oaesfd[1]);
return -1;
} else if (oaes_pid == 0) {
// Child
close(oaesfd[1]); // close unused
int output_fd = open(tarfn.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH);
if (output_fd < 0) {
LOGERR("Failed to open '%s'\n", tarfn.c_str());
_exit(-1);
}
dup2(oaesfd[0], 0); // remap stdin
dup2(output_fd, 1); // remap stdout to output file
if (execlp("openaes", "openaes", "enc", "--key", Password.c_str(), NULL) < 0) {
LOGERR("execlp openaes ERROR!\n");
close(output_fd);
close(oaesfd[0]);
_exit(-1);
}
} else {
// Parent
close(oaesfd[0]); // close parent input
fd = oaesfd[1]; // copy parent output
if(tar_fdopen(&t, fd, charRootDir, NULL, O_WRONLY | O_CREAT | O_EXCL | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
close(fd);
LOGERR("tar_fdopen failed\n");
return -1;
}
return 0;
}
} else {
// Not compressed or encrypted
init_libtar_buffer(0);
if (tar_open(&t, charTarFile, &type, O_WRONLY | O_CREAT | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) == -1) {
LOGERR("tar_open error opening '%s'\n", tarfn.c_str());
return -1;
}
}
return 0;
}
int twrpTar::openTar() {
char* charRootDir = (char*) tardir.c_str();
char* charTarFile = (char*) tarfn.c_str();
string Password;
if (Archive_Current_Type == 3) {
LOGINFO("Opening encrypted and compressed backup...\n");
DataManager::GetValue("tw_restore_password", Password);
int i, pipes[4];
if (pipe(pipes) < 0) {
LOGERR("Error creating first pipe\n");
return -1;
}
if (pipe(pipes + 2) < 0) {
LOGERR("Error creating second pipe\n");
return -1;
}
oaes_pid = fork();
if (oaes_pid < 0) {
LOGERR("pigz fork() failed\n");
for (i = 0; i < 4; i++)
close(pipes[i]); // close all
return -1;
} else if (oaes_pid == 0) {
// openaes Child
close(pipes[0]); // Close pipes that are not used by this child
close(pipes[2]);
close(pipes[3]);
int input_fd = open(tarfn.c_str(), O_RDONLY | O_LARGEFILE);
if (input_fd < 0) {
LOGERR("Failed to open '%s'\n", tarfn.c_str());
close(pipes[1]);
_exit(-1);
}
close(0);
dup2(input_fd, 0);
close(1);
dup2(pipes[1], 1);
if (execlp("openaes", "openaes", "dec", "--key", Password.c_str(), NULL) < 0) {
LOGERR("execlp openaes ERROR!\n");
close(input_fd);
close(pipes[1]);
_exit(-1);
}
} else {
// Parent
pigz_pid = fork();
if (pigz_pid < 0) {
LOGERR("openaes fork() failed\n");
for (i = 0; i < 4; i++)
close(pipes[i]); // close all
return -1;
} else if (pigz_pid == 0) {
// pigz Child
close(pipes[1]); // Close pipes not used by this child
close(pipes[2]);
close(0);
dup2(pipes[0], 0);
close(1);
dup2(pipes[3], 1);
if (execlp("pigz", "pigz", "-d", "-c", NULL) < 0) {
LOGERR("execlp pigz ERROR!\n");
close(pipes[0]);
close(pipes[3]);
_exit(-1);
}
} else {
// Parent
close(pipes[0]); // Close pipes not used by parent
close(pipes[1]);
close(pipes[3]);
fd = pipes[2];
if(tar_fdopen(&t, fd, charRootDir, NULL, O_RDONLY | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
close(fd);
LOGERR("tar_fdopen failed\n");
return -1;
}
}
}
} else if (Archive_Current_Type == 2) {
LOGINFO("Opening encrypted backup...\n");
DataManager::GetValue("tw_restore_password", Password);
int oaesfd[2];
pipe(oaesfd);
oaes_pid = fork();
if (oaes_pid < 0) {
LOGERR("fork() failed\n");
close(oaesfd[0]);
close(oaesfd[1]);
return -1;
} else if (oaes_pid == 0) {
// Child
close(oaesfd[0]); // Close unused pipe
int input_fd = open(tarfn.c_str(), O_RDONLY | O_LARGEFILE);
if (input_fd < 0) {
LOGERR("Failed to open '%s'\n", tarfn.c_str());
close(oaesfd[1]);
_exit(-1);
}
close(0); // close stdin
dup2(oaesfd[1], 1); // remap stdout
dup2(input_fd, 0); // remap input fd to stdin
if (execlp("openaes", "openaes", "dec", "--key", Password.c_str(), NULL) < 0) {
LOGERR("execlp openaes ERROR!\n");
close(input_fd);
close(oaesfd[1]);
_exit(-1);
}
} else {
// Parent
close(oaesfd[1]); // close parent output
fd = oaesfd[0]; // copy parent input
if(tar_fdopen(&t, fd, charRootDir, NULL, O_RDONLY | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
close(fd);
LOGERR("tar_fdopen failed\n");
return -1;
}
}
} else if (Archive_Current_Type == 1) {
LOGINFO("Opening as a gzip...\n");
int pigzfd[2];
pipe(pigzfd);
pigz_pid = fork();
if (pigz_pid < 0) {
LOGERR("fork() failed\n");
close(pigzfd[0]);
close(pigzfd[1]);
return -1;
} else if (pigz_pid == 0) {
// Child
close(pigzfd[0]);
int input_fd = open(tarfn.c_str(), O_RDONLY | O_LARGEFILE);
if (input_fd < 0) {
LOGERR("Failed to open '%s'\n", tarfn.c_str());
_exit(-1);
}
dup2(input_fd, 0); // remap input fd to stdin
dup2(pigzfd[1], 1); // remap stdout
if (execlp("pigz", "pigz", "-d", "-c", NULL) < 0) {
close(pigzfd[1]);
close(input_fd);
LOGERR("execlp openaes ERROR!\n");
_exit(-1);
}
} else {
// Parent
close(pigzfd[1]); // close parent output
fd = pigzfd[0]; // copy parent input
if(tar_fdopen(&t, fd, charRootDir, NULL, O_RDONLY | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
close(fd);
LOGERR("tar_fdopen failed\n");
return -1;
}
}
} else if (tar_open(&t, charTarFile, NULL, O_RDONLY | O_LARGEFILE, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH, TAR_GNU | TAR_STORE_SELINUX) != 0) {
LOGERR("Unable to open tar archive '%s'\n", charTarFile);
return -1;
}
return 0;
}
string twrpTar::Strip_Root_Dir(string Path) {
string temp;
size_t slash;
if (Path.substr(0, 1) == "/")
temp = Path.substr(1, Path.size() - 1);
else
temp = Path;
slash = temp.find("/");
if (slash == string::npos)
return temp;
else {
string stripped;
stripped = temp.substr(slash, temp.size() - slash);
return stripped;
}
return temp;
}
int twrpTar::addFile(string fn, bool include_root) {
char* charTarFile = (char*) fn.c_str();
if (include_root) {
if (tar_append_file(t, charTarFile, NULL) == -1)
return -1;
} else {
string temp = Strip_Root_Dir(fn);
char* charTarPath = (char*) temp.c_str();
if (tar_append_file(t, charTarFile, charTarPath) == -1)
return -1;
}
return 0;
}
int twrpTar::closeTar() {
flush_libtar_buffer(t->fd);
if (tar_append_eof(t) != 0) {
LOGERR("tar_append_eof(): %s\n", strerror(errno));
tar_close(t);
return -1;
}
if (tar_close(t) != 0) {
LOGERR("Unable to close tar archive: '%s'\n", tarfn.c_str());
return -1;
}
if (Archive_Current_Type > 0) {
close(fd);
int status;
if (pigz_pid > 0 && TWFunc::Wait_For_Child(pigz_pid, &status, "pigz") != 0)
return -1;
if (oaes_pid > 0 && TWFunc::Wait_For_Child(oaes_pid, &status, "openaes") != 0)
return -1;
}
free_libtar_buffer();
return 0;
}
int twrpTar::removeEOT(string tarFile) {
char* charTarFile = (char*) tarFile.c_str();
off_t tarFileEnd;
while (th_read(t) == 0) {
if (TH_ISREG(t))
tar_skip_regfile(t);
tarFileEnd = lseek(t->fd, 0, SEEK_CUR);
}
if (tar_close(t) == -1)
return -1;
if (truncate(charTarFile, tarFileEnd) == -1)
return -1;
return 0;
}
int twrpTar::entryExists(string entry) {
char* searchstr = (char*)entry.c_str();
int ret;
Archive_Current_Type = TWFunc::Get_File_Type(tarfn);
if (openTar() == -1)
ret = 0;
else
ret = tar_find(t, searchstr);
if (closeTar() != 0)
LOGINFO("Unable to close tar after searching for entry.\n");
return ret;
}
unsigned long long twrpTar::uncompressedSize() {
int type = 0;
unsigned long long total_size = 0;
string Tar, Command, result;
vector<string> split;
Tar = TWFunc::Get_Filename(tarfn);
type = TWFunc::Get_File_Type(tarfn);
if (type == 0)
total_size = TWFunc::Get_File_Size(tarfn);
else {
Command = "pigz -l " + tarfn;
/* if we set Command = "pigz -l " + tarfn + " | sed '1d' | cut -f5 -d' '";
we get the uncompressed size at once. */
TWFunc::Exec_Cmd(Command, result);
if (!result.empty()) {
/* Expected output:
compressed original reduced name
95855838 179403776 -1.3% data.yaffs2.win
^
split[5]
*/
split = TWFunc::split_string(result, ' ', true);
if (split.size() > 4)
total_size = atoi(split[5].c_str());
}
}
LOGINFO("%s's uncompressed size: %llu bytes\n", Tar.c_str(), total_size);
return total_size;
}
extern "C" ssize_t write_tar(int fd, const void *buffer, size_t size) {
return (ssize_t) write_libtar_buffer(fd, buffer, size);
}
Reference in New Issue View Git Blame Copy Permalink