From d29f641b1c4809c0f074a179286e163d376a289e Mon Sep 17 00:00:00 2001 From: Michael Runge Date: Mon, 21 Jul 2014 17:40:02 -0700 Subject: [PATCH 01/13] Auto create parent directories for rename support Sometimes renames will move a file into a directory that does not yet exist. This will create the parent directories, using the same symlink logic, to ensure that there is a valid destination. Bug: 16458395 Change-Id: Iaa005a12ce800c39f4db20f7c25a2a68cb40a52d --- updater/install.c | 8 +++++--- updater/install.h | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/updater/install.c b/updater/install.c index edc386dc..5025881d 100644 --- a/updater/install.c +++ b/updater/install.c @@ -357,8 +357,10 @@ Value* RenameFn(const char* name, State* state, int argc, Expr* argv[]) { name); goto done; } - - if (rename(src_name, dst_name) != 0) { + if (make_parents(dst_name) != 0) { + ErrorAbort(state, "Creating parent of %s() failed, error %s()", + dst_name, strerror(errno)); + } else if (rename(src_name, dst_name) != 0) { ErrorAbort(state, "Rename of %s() to %s() failed, error %s()", src_name, dst_name, strerror(errno)); } else { @@ -642,7 +644,7 @@ static int make_parents(char* name) { *p = '\0'; if (make_parents(name) < 0) return -1; int result = mkdir(name, 0700); - if (result == 0) printf("symlink(): created [%s]\n", name); + if (result == 0) printf("created [%s]\n", name); *p = '/'; if (result == 0 || errno == EEXIST) { // successfully created or already existed; we're done diff --git a/updater/install.h b/updater/install.h index 94f344f8..659c8b41 100644 --- a/updater/install.h +++ b/updater/install.h @@ -19,4 +19,6 @@ void RegisterInstallFunctions(); +static int make_parents(char* name); + #endif From 66f5ce387af87584b40eeec774429d7ead28b7cc Mon Sep 17 00:00:00 2001 From: Doug Zongker Date: Fri, 15 Aug 2014 14:31:52 -0700 Subject: [PATCH 02/13] installer for new block OTA system Bug: 16984795 Change-Id: I90f958446baed83dec658de2430c8fc5e9c3047e --- applypatch/applypatch.c | 6 +- applypatch/applypatch.h | 2 +- applypatch/bspatch.c | 4 +- applypatch/imgpatch.c | 4 +- updater/Android.mk | 1 + updater/blockimg.c | 631 ++++++++++++++++++++++++++++++++++++++++ updater/blockimg.h | 22 ++ updater/install.c | 2 +- updater/updater.c | 4 + updater/updater.h | 3 + 10 files changed, 669 insertions(+), 10 deletions(-) create mode 100644 updater/blockimg.c create mode 100644 updater/blockimg.h diff --git a/applypatch/applypatch.c b/applypatch/applypatch.c index 60e9e4a5..bfb9440e 100644 --- a/applypatch/applypatch.c +++ b/applypatch/applypatch.c @@ -32,7 +32,7 @@ #include "edify/expr.h" static int LoadPartitionContents(const char* filename, FileContents* file); -static ssize_t FileSink(unsigned char* data, ssize_t len, void* token); +static ssize_t FileSink(const unsigned char* data, ssize_t len, void* token); static int GenerateTarget(FileContents* source_file, const Value* source_patch_value, FileContents* copy_file, @@ -599,7 +599,7 @@ int ShowLicenses() { return 0; } -ssize_t FileSink(unsigned char* data, ssize_t len, void* token) { +ssize_t FileSink(const unsigned char* data, ssize_t len, void* token) { int fd = *(int *)token; ssize_t done = 0; ssize_t wrote; @@ -620,7 +620,7 @@ typedef struct { ssize_t pos; } MemorySinkInfo; -ssize_t MemorySink(unsigned char* data, ssize_t len, void* token) { +ssize_t MemorySink(const unsigned char* data, ssize_t len, void* token) { MemorySinkInfo* msi = (MemorySinkInfo*)token; if (msi->size - msi->pos < len) { return -1; diff --git a/applypatch/applypatch.h b/applypatch/applypatch.h index ee54c24e..edec8481 100644 --- a/applypatch/applypatch.h +++ b/applypatch/applypatch.h @@ -40,7 +40,7 @@ typedef struct _FileContents { // and use it as the source instead. #define CACHE_TEMP_SOURCE "/cache/saved.file" -typedef ssize_t (*SinkFn)(unsigned char*, ssize_t, void*); +typedef ssize_t (*SinkFn)(const unsigned char*, ssize_t, void*); // applypatch.c int ShowLicenses(); diff --git a/applypatch/bspatch.c b/applypatch/bspatch.c index 1dc7ab10..b34ec2a8 100644 --- a/applypatch/bspatch.c +++ b/applypatch/bspatch.c @@ -112,9 +112,7 @@ int ApplyBSDiffPatch(const unsigned char* old_data, ssize_t old_size, printf("short write of output: %d (%s)\n", errno, strerror(errno)); return 1; } - if (ctx) { - SHA_update(ctx, new_data, new_size); - } + if (ctx) SHA_update(ctx, new_data, new_size); free(new_data); return 0; diff --git a/applypatch/imgpatch.c b/applypatch/imgpatch.c index af4d0728..33c44876 100644 --- a/applypatch/imgpatch.c +++ b/applypatch/imgpatch.c @@ -95,7 +95,7 @@ int ApplyImagePatch(const unsigned char* old_data, ssize_t old_size __unused, printf("failed to read chunk %d raw data\n", i); return -1; } - SHA_update(ctx, patch->data + pos, data_len); + if (ctx) SHA_update(ctx, patch->data + pos, data_len); if (sink((unsigned char*)patch->data + pos, data_len, token) != data_len) { printf("failed to write chunk %d raw data\n", i); @@ -217,7 +217,7 @@ int ApplyImagePatch(const unsigned char* old_data, ssize_t old_size __unused, (long)have); return -1; } - SHA_update(ctx, temp_data, have); + if (ctx) SHA_update(ctx, temp_data, have); } while (ret != Z_STREAM_END); deflateEnd(&strm); diff --git a/updater/Android.mk b/updater/Android.mk index 99b48902..b183c922 100644 --- a/updater/Android.mk +++ b/updater/Android.mk @@ -4,6 +4,7 @@ LOCAL_PATH := $(call my-dir) updater_src_files := \ install.c \ + blockimg.c \ updater.c # diff --git a/updater/blockimg.c b/updater/blockimg.c new file mode 100644 index 00000000..c442ab22 --- /dev/null +++ b/updater/blockimg.c @@ -0,0 +1,631 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "applypatch/applypatch.h" +#include "edify/expr.h" +#include "mincrypt/sha.h" +#include "minzip/DirUtil.h" +#include "updater.h" + +#define BLOCKSIZE 4096 + +// Set this to 1 to interpret 'erase' transfers to mean do a +// BLKDISCARD ioctl. Set to 0 to interpret erase to mean fill the +// region with zeroes. +#define DEBUG_ERASE 0 + +#ifndef BLKDISCARD +#define BLKDISCARD _IO(0x12,119) +#endif + +char* PrintSha1(const uint8_t* digest); + +typedef struct { + int count; + int size; + int pos[0]; +} RangeSet; + +static RangeSet* parse_range(char* text) { + char* save; + int num; + num = strtol(strtok_r(text, ",", &save), NULL, 0); + + RangeSet* out = malloc(sizeof(RangeSet) + num * sizeof(int)); + if (out == NULL) { + fprintf(stderr, "failed to allocate range of %d bytes\n", + sizeof(RangeSet) + num * sizeof(int)); + exit(1); + } + out->count = num / 2; + out->size = 0; + int i; + for (i = 0; i < num; ++i) { + out->pos[i] = strtol(strtok_r(NULL, ",", &save), NULL, 0); + if (i%2) { + out->size += out->pos[i]; + } else { + out->size -= out->pos[i]; + } + } + + return out; +} + +static void readblock(int fd, uint8_t* data, size_t size) { + size_t so_far = 0; + while (so_far < size) { + ssize_t r = read(fd, data+so_far, size-so_far); + if (r < 0 && errno != EINTR) { + fprintf(stderr, "read failed: %s\n", strerror(errno)); + return; + } else { + so_far += r; + } + } +} + +static void writeblock(int fd, const uint8_t* data, size_t size) { + size_t written = 0; + while (written < size) { + ssize_t w = write(fd, data+written, size-written); + if (w < 0 && errno != EINTR) { + fprintf(stderr, "write failed: %s\n", strerror(errno)); + return; + } else { + written += w; + } + } +} + +static void check_lseek(int fd, off_t offset, int whence) { + while (true) { + int ret = lseek(fd, offset, whence); + if (ret < 0) { + if (errno != EINTR) { + fprintf(stderr, "lseek failed: %s\n", strerror(errno)); + exit(1); + } + } else { + break; + } + } +} + +static void allocate(size_t size, uint8_t** buffer, size_t* buffer_alloc) { + // if the buffer's big enough, reuse it. + if (size <= *buffer_alloc) return; + + free(*buffer); + + *buffer = (uint8_t*) malloc(size); + if (*buffer == NULL) { + fprintf(stderr, "failed to allocate %d bytes\n", size); + exit(1); + } + *buffer_alloc = size; +} + +typedef struct { + int fd; + RangeSet* tgt; + int p_block; + size_t p_remain; +} RangeSinkState; + +static ssize_t RangeSinkWrite(const uint8_t* data, ssize_t size, void* token) { + RangeSinkState* rss = (RangeSinkState*) token; + + if (rss->p_remain <= 0) { + fprintf(stderr, "range sink write overrun"); + exit(1); + } + + ssize_t written = 0; + while (size > 0) { + size_t write_now = size; + if (rss->p_remain < write_now) write_now = rss->p_remain; + writeblock(rss->fd, data, write_now); + data += write_now; + size -= write_now; + + rss->p_remain -= write_now; + written += write_now; + + if (rss->p_remain == 0) { + // move to the next block + ++rss->p_block; + if (rss->p_block < rss->tgt->count) { + rss->p_remain = (rss->tgt->pos[rss->p_block*2+1] - rss->tgt->pos[rss->p_block*2]) * BLOCKSIZE; + check_lseek(rss->fd, rss->tgt->pos[rss->p_block*2] * BLOCKSIZE, SEEK_SET); + } else { + // we can't write any more; return how many bytes have + // been written so far. + return written; + } + } + } + + return written; +} + +// All of the data for all the 'new' transfers is contained in one +// file in the update package, concatenated together in the order in +// which transfers.list will need it. We want to stream it out of the +// archive (it's compressed) without writing it to a temp file, but we +// can't write each section until it's that transfer's turn to go. +// +// To achieve this, we expand the new data from the archive in a +// background thread, and block that threads 'receive uncompressed +// data' function until the main thread has reached a point where we +// want some new data to be written. We signal the background thread +// with the destination for the data and block the main thread, +// waiting for the background thread to complete writing that section. +// Then it signals the main thread to wake up and goes back to +// blocking waiting for a transfer. +// +// NewThreadInfo is the struct used to pass information back and forth +// between the two threads. When the main thread wants some data +// written, it sets rss to the destination location and signals the +// condition. When the background thread is done writing, it clears +// rss and signals the condition again. + +typedef struct { + ZipArchive* za; + const ZipEntry* entry; + + RangeSinkState* rss; + + pthread_mutex_t mu; + pthread_cond_t cv; +} NewThreadInfo; + +static bool receive_new_data(const unsigned char* data, int size, void* cookie) { + NewThreadInfo* nti = (NewThreadInfo*) cookie; + + while (size > 0) { + // Wait for nti->rss to be non-NULL, indicating some of this + // data is wanted. + pthread_mutex_lock(&nti->mu); + while (nti->rss == NULL) { + pthread_cond_wait(&nti->cv, &nti->mu); + } + pthread_mutex_unlock(&nti->mu); + + // At this point nti->rss is set, and we own it. The main + // thread is waiting for it to disappear from nti. + ssize_t written = RangeSinkWrite(data, size, nti->rss); + data += written; + size -= written; + + if (nti->rss->p_block == nti->rss->tgt->count) { + // we have written all the bytes desired by this rss. + + pthread_mutex_lock(&nti->mu); + nti->rss = NULL; + pthread_cond_broadcast(&nti->cv); + pthread_mutex_unlock(&nti->mu); + } + } + + return true; +} + +static void* unzip_new_data(void* cookie) { + NewThreadInfo* nti = (NewThreadInfo*) cookie; + mzProcessZipEntryContents(nti->za, nti->entry, receive_new_data, nti); + return NULL; +} + +// args: +// - block device (or file) to modify in-place +// - transfer list (blob) +// - new data stream (filename within package.zip) +// - patch stream (filename within package.zip, must be uncompressed) + +Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[]) { + Value* blockdev_filename; + Value* transfer_list; + Value* new_data_fn; + Value* patch_data_fn; + bool success = false; + + if (ReadValueArgs(state, argv, 4, &blockdev_filename, &transfer_list, + &new_data_fn, &patch_data_fn) < 0) { + return NULL; + } + + if (blockdev_filename->type != VAL_STRING) { + ErrorAbort(state, "blockdev_filename argument to %s must be string", name); + goto done; + } + if (transfer_list->type != VAL_BLOB) { + ErrorAbort(state, "transfer_list argument to %s must be blob", name); + goto done; + } + if (new_data_fn->type != VAL_STRING) { + ErrorAbort(state, "new_data_fn argument to %s must be string", name); + goto done; + } + if (patch_data_fn->type != VAL_STRING) { + ErrorAbort(state, "patch_data_fn argument to %s must be string", name); + goto done; + } + + UpdaterInfo* ui = (UpdaterInfo*)(state->cookie); + FILE* cmd_pipe = ui->cmd_pipe; + + ZipArchive* za = ((UpdaterInfo*)(state->cookie))->package_zip; + + const ZipEntry* patch_entry = mzFindZipEntry(za, patch_data_fn->data); + if (patch_entry == NULL) { + ErrorAbort(state, "%s(): no file \"%s\" in package", name, patch_data_fn->data); + goto done; + } + + uint8_t* patch_start = ((UpdaterInfo*)(state->cookie))->package_zip_addr + + mzGetZipEntryOffset(patch_entry); + + const ZipEntry* new_entry = mzFindZipEntry(za, new_data_fn->data); + if (new_entry == NULL) { + ErrorAbort(state, "%s(): no file \"%s\" in package", name, new_data_fn->data); + goto done; + } + + // The transfer list is a text file containing commands to + // transfer data from one place to another on the target + // partition. We parse it and execute the commands in order: + // + // zero [rangeset] + // - fill the indicated blocks with zeros + // + // new [rangeset] + // - fill the blocks with data read from the new_data file + // + // bsdiff patchstart patchlen [src rangeset] [tgt rangeset] + // imgdiff patchstart patchlen [src rangeset] [tgt rangeset] + // - read the source blocks, apply a patch, write result to + // target blocks. bsdiff or imgdiff specifies the type of + // patch. + // + // move [src rangeset] [tgt rangeset] + // - copy data from source blocks to target blocks (no patch + // needed; rangesets are the same size) + // + // erase [rangeset] + // - mark the given blocks as empty + // + // The creator of the transfer list will guarantee that no block + // is read (ie, used as the source for a patch or move) after it + // has been written. + // + // Within one command the source and target ranges may overlap so + // in general we need to read the entire source into memory before + // writing anything to the target blocks. + // + // All the patch data is concatenated into one patch_data file in + // the update package. It must be stored uncompressed because we + // memory-map it in directly from the archive. (Since patches are + // already compressed, we lose very little by not compressing + // their concatenation.) + + pthread_t new_data_thread; + NewThreadInfo nti; + nti.za = za; + nti.entry = new_entry; + nti.rss = NULL; + pthread_mutex_init(&nti.mu, NULL); + pthread_cond_init(&nti.cv, NULL); + + pthread_attr_t attr; + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + pthread_create(&new_data_thread, &attr, unzip_new_data, &nti); + + int i, j; + + char* linesave; + char* wordsave; + + int fd = open(blockdev_filename->data, O_RDWR); + if (fd < 0) { + ErrorAbort(state, "failed to open %s: %s", blockdev_filename->data, strerror(errno)); + goto done; + } + + char* line; + char* word; + + line = strtok_r(transfer_list->data, "\n", &linesave); + + // first line in transfer list is the version number; currently + // there's only version 1. + if (strcmp(line, "1") != 0) { + ErrorAbort(state, "unexpected transfer list version [%s]\n", line); + goto done; + } + + // second line in transfer list is the total number of blocks we + // expect to write. + line = strtok_r(NULL, "\n", &linesave); + int total_blocks = strtol(line, NULL, 0); + // shouldn't happen, but avoid divide by zero. + if (total_blocks == 0) ++total_blocks; + int blocks_so_far = 0; + + uint8_t* buffer = NULL; + size_t buffer_alloc = 0; + + // third and subsequent lines are all individual transfer commands. + for (line = strtok_r(NULL, "\n", &linesave); line; + line = strtok_r(NULL, "\n", &linesave)) { + char* style; + style = strtok_r(line, " ", &wordsave); + + if (strcmp("move", style) == 0) { + word = strtok_r(NULL, " ", &wordsave); + RangeSet* src = parse_range(word); + word = strtok_r(NULL, " ", &wordsave); + RangeSet* tgt = parse_range(word); + + printf(" moving %d blocks\n", src->size); + + allocate(src->size * BLOCKSIZE, &buffer, &buffer_alloc); + size_t p = 0; + for (i = 0; i < src->count; ++i) { + check_lseek(fd, src->pos[i*2] * BLOCKSIZE, SEEK_SET); + size_t sz = (src->pos[i*2+1] - src->pos[i*2]) * BLOCKSIZE; + readblock(fd, buffer+p, sz); + p += sz; + } + + p = 0; + for (i = 0; i < tgt->count; ++i) { + check_lseek(fd, tgt->pos[i*2] * BLOCKSIZE, SEEK_SET); + size_t sz = (tgt->pos[i*2+1] - tgt->pos[i*2]) * BLOCKSIZE; + writeblock(fd, buffer+p, sz); + p += sz; + } + + blocks_so_far += tgt->size; + fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); + fflush(cmd_pipe); + + free(src); + free(tgt); + + } else if (strcmp("zero", style) == 0 || + (DEBUG_ERASE && strcmp("erase", style) == 0)) { + word = strtok_r(NULL, " ", &wordsave); + RangeSet* tgt = parse_range(word); + + printf(" zeroing %d blocks\n", tgt->size); + + allocate(BLOCKSIZE, &buffer, &buffer_alloc); + memset(buffer, 0, BLOCKSIZE); + for (i = 0; i < tgt->count; ++i) { + check_lseek(fd, tgt->pos[i*2] * BLOCKSIZE, SEEK_SET); + for (j = tgt->pos[i*2]; j < tgt->pos[i*2+1]; ++j) { + writeblock(fd, buffer, BLOCKSIZE); + } + } + + if (style[0] == 'z') { // "zero" but not "erase" + blocks_so_far += tgt->size; + fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); + fflush(cmd_pipe); + } + + free(tgt); + } else if (strcmp("new", style) == 0) { + + word = strtok_r(NULL, " ", &wordsave); + RangeSet* tgt = parse_range(word); + + printf(" writing %d blocks of new data\n", tgt->size); + + RangeSinkState rss; + rss.fd = fd; + rss.tgt = tgt; + rss.p_block = 0; + rss.p_remain = (tgt->pos[1] - tgt->pos[0]) * BLOCKSIZE; + check_lseek(fd, tgt->pos[0] * BLOCKSIZE, SEEK_SET); + + pthread_mutex_lock(&nti.mu); + nti.rss = &rss; + pthread_cond_broadcast(&nti.cv); + while (nti.rss) { + pthread_cond_wait(&nti.cv, &nti.mu); + } + pthread_mutex_unlock(&nti.mu); + + blocks_so_far += tgt->size; + fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); + fflush(cmd_pipe); + + free(tgt); + + } else if (strcmp("bsdiff", style) == 0 || + strcmp("imgdiff", style) == 0) { + word = strtok_r(NULL, " ", &wordsave); + size_t patch_offset = strtoul(word, NULL, 0); + word = strtok_r(NULL, " ", &wordsave); + size_t patch_len = strtoul(word, NULL, 0); + + word = strtok_r(NULL, " ", &wordsave); + RangeSet* src = parse_range(word); + word = strtok_r(NULL, " ", &wordsave); + RangeSet* tgt = parse_range(word); + + printf(" patching %d blocks to %d\n", src->size, tgt->size); + + // Read the source into memory. + allocate(src->size * BLOCKSIZE, &buffer, &buffer_alloc); + size_t p = 0; + for (i = 0; i < src->count; ++i) { + check_lseek(fd, src->pos[i*2] * BLOCKSIZE, SEEK_SET); + size_t sz = (src->pos[i*2+1] - src->pos[i*2]) * BLOCKSIZE; + readblock(fd, buffer+p, sz); + p += sz; + } + + Value patch_value; + patch_value.type = VAL_BLOB; + patch_value.size = patch_len; + patch_value.data = (char*)(patch_start + patch_offset); + + RangeSinkState rss; + rss.fd = fd; + rss.tgt = tgt; + rss.p_block = 0; + rss.p_remain = (tgt->pos[1] - tgt->pos[0]) * BLOCKSIZE; + check_lseek(fd, tgt->pos[0] * BLOCKSIZE, SEEK_SET); + + if (style[0] == 'i') { // imgdiff + ApplyImagePatch(buffer, src->size * BLOCKSIZE, + &patch_value, + &RangeSinkWrite, &rss, NULL, NULL); + } else { + ApplyBSDiffPatch(buffer, src->size * BLOCKSIZE, + &patch_value, 0, + &RangeSinkWrite, &rss, NULL); + } + + // We expect the output of the patcher to fill the tgt ranges exactly. + if (rss.p_block != tgt->count || rss.p_remain != 0) { + fprintf(stderr, "range sink underrun?\n"); + } + + blocks_so_far += tgt->size; + fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); + fflush(cmd_pipe); + + free(src); + free(tgt); + } else if (!DEBUG_ERASE && strcmp("erase", style) == 0) { + struct stat st; + if (fstat(fd, &st) == 0 && S_ISBLK(st.st_mode)) { + word = strtok_r(NULL, " ", &wordsave); + RangeSet* tgt = parse_range(word); + + printf(" erasing %d blocks\n", tgt->size); + + for (i = 0; i < tgt->count; ++i) { + uint64_t range[2]; + // offset in bytes + range[0] = tgt->pos[i*2] * BLOCKSIZE; + // len in bytes + range[1] = (tgt->pos[i*2+1] - tgt->pos[i*2]) * BLOCKSIZE; + + if (ioctl(fd, BLKDISCARD, &range) < 0) { + printf(" blkdiscard failed: %s\n", strerror(errno)); + } + } + + free(tgt); + } else { + printf(" ignoring erase (not block device)\n"); + } + } else { + fprintf(stderr, "unknown transfer style \"%s\"\n", style); + exit(1); + } + } + + pthread_join(new_data_thread, NULL); + success = true; + + free(buffer); + printf("wrote %d blocks; expected %d\n", blocks_so_far, total_blocks); + printf("max alloc needed was %zu\n", buffer_alloc); + + done: + FreeValue(blockdev_filename); + FreeValue(transfer_list); + FreeValue(new_data_fn); + FreeValue(patch_data_fn); + return StringValue(success ? strdup("t") : strdup("")); +} + +Value* RangeSha1Fn(const char* name, State* state, int argc, Expr* argv[]) { + Value* blockdev_filename; + Value* ranges; + const uint8_t* digest = NULL; + if (ReadValueArgs(state, argv, 2, &blockdev_filename, &ranges) < 0) { + return NULL; + } + + if (blockdev_filename->type != VAL_STRING) { + ErrorAbort(state, "blockdev_filename argument to %s must be string", name); + goto done; + } + if (ranges->type != VAL_STRING) { + ErrorAbort(state, "ranges argument to %s must be string", name); + goto done; + } + + int fd = open(blockdev_filename->data, O_RDWR); + if (fd < 0) { + ErrorAbort(state, "failed to open %s: %s", blockdev_filename->data, strerror(errno)); + goto done; + } + + RangeSet* rs = parse_range(ranges->data); + uint8_t buffer[BLOCKSIZE]; + + SHA_CTX ctx; + SHA_init(&ctx); + + int i, j; + for (i = 0; i < rs->count; ++i) { + check_lseek(fd, rs->pos[i*2] * BLOCKSIZE, SEEK_SET); + for (j = rs->pos[i*2]; j < rs->pos[i*2+1]; ++j) { + readblock(fd, buffer, BLOCKSIZE); + SHA_update(&ctx, buffer, BLOCKSIZE); + } + } + digest = SHA_final(&ctx); + close(fd); + + done: + FreeValue(blockdev_filename); + FreeValue(ranges); + if (digest == NULL) { + return StringValue(strdup("")); + } else { + return StringValue(PrintSha1(digest)); + } +} + +void RegisterBlockImageFunctions() { + RegisterFunction("block_image_update", BlockImageUpdateFn); + RegisterFunction("range_sha1", RangeSha1Fn); +} diff --git a/updater/blockimg.h b/updater/blockimg.h new file mode 100644 index 00000000..2f4ad3c0 --- /dev/null +++ b/updater/blockimg.h @@ -0,0 +1,22 @@ +/* + * Copyright (C) 2014 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef _UPDATER_BLOCKIMG_H_ +#define _UPDATER_BLOCKIMG_H_ + +void RegisterBlockImageFunctions(); + +#endif diff --git a/updater/install.c b/updater/install.c index 19861800..cdcdb8fd 100644 --- a/updater/install.c +++ b/updater/install.c @@ -54,7 +54,7 @@ #endif // Take a sha-1 digest and return it as a newly-allocated hex string. -static char* PrintSha1(const uint8_t* digest) { +char* PrintSha1(const uint8_t* digest) { char* buffer = malloc(SHA_DIGEST_SIZE*2 + 1); int i; const char* alphabet = "0123456789abcdef"; diff --git a/updater/updater.c b/updater/updater.c index b7af3e50..465e1238 100644 --- a/updater/updater.c +++ b/updater/updater.c @@ -21,6 +21,7 @@ #include "edify/expr.h" #include "updater.h" #include "install.h" +#include "blockimg.h" #include "minzip/Zip.h" #include "minzip/SysUtil.h" @@ -98,6 +99,7 @@ int main(int argc, char** argv) { RegisterBuiltins(); RegisterInstallFunctions(); + RegisterBlockImageFunctions(); RegisterDeviceExtensions(); FinishRegistration(); @@ -127,6 +129,8 @@ int main(int argc, char** argv) { updater_info.cmd_pipe = cmd_pipe; updater_info.package_zip = &za; updater_info.version = atoi(version); + updater_info.package_zip_addr = map.addr; + updater_info.package_zip_len = map.length; State state; state.cookie = &updater_info; diff --git a/updater/updater.h b/updater/updater.h index d2e90114..d1dfdd05 100644 --- a/updater/updater.h +++ b/updater/updater.h @@ -27,6 +27,9 @@ typedef struct { FILE* cmd_pipe; ZipArchive* package_zip; int version; + + uint8_t* package_zip_addr; + size_t package_zip_len; } UpdaterInfo; extern struct selabel_handle *sehandle; From adad8ec045e118215b1745c2ba86031c7f2c73d2 Mon Sep 17 00:00:00 2001 From: Doug Zongker Date: Fri, 22 Aug 2014 14:53:43 -0700 Subject: [PATCH 03/13] remove code for original block OTA mechanism Superseded by newer code. Bug: 16984795 Change-Id: I70c1d29dc03287b06ea909d17f729ec51ccb0344 --- updater/Android.mk | 2 - updater/install.c | 213 ++------------------------------------------- 2 files changed, 6 insertions(+), 209 deletions(-) diff --git a/updater/Android.mk b/updater/Android.mk index b183c922..a3a900a8 100644 --- a/updater/Android.mk +++ b/updater/Android.mk @@ -36,8 +36,6 @@ LOCAL_STATIC_LIBRARIES += libcutils liblog libstdc++ libc LOCAL_STATIC_LIBRARIES += libselinux LOCAL_C_INCLUDES += $(LOCAL_PATH)/.. -LOCAL_STATIC_LIBRARIES += libsyspatch libxz libxdelta3 - # Each library in TARGET_RECOVERY_UPDATER_LIBS should have a function # named "Register_()". Here we emit a little C function that # gets #included by updater.c. It calls all those registration diff --git a/updater/install.c b/updater/install.c index cdcdb8fd..dad0d08c 100644 --- a/updater/install.c +++ b/updater/install.c @@ -45,7 +45,6 @@ #include "mtdutils/mounts.h" #include "mtdutils/mtdutils.h" #include "updater.h" -#include "syspatch.h" #include "install.h" #ifdef USE_EXT4 @@ -464,68 +463,6 @@ Value* PackageExtractDirFn(const char* name, State* state, } -DontCareMap* ReadDontCareMapFromZip(ZipArchive* za, const char* path) { - const char* name = "ReadDontCareMapFromZip"; - - const ZipEntry* entry = mzFindZipEntry(za, path); - if (entry == NULL) { - printf("%s: no %s in package\n", name, path); - return NULL; - } - - size_t map_size = mzGetZipEntryUncompLen(entry); - char* map_data = malloc(map_size); - if (map_data == NULL) { - printf("%s: failed to allocate %zu bytes for %s\n", - name, map_size, path); - return NULL; - } - - if (!mzExtractZipEntryToBuffer(za, entry, (unsigned char*) map_data)) { - printf("%s: failed to read %s\n", name, path); - return NULL; - } - - char* p = map_data; - DontCareMap* map = (DontCareMap*) malloc(sizeof(DontCareMap)); - - map->block_size = strtoul(p, &p, 0); - if (map->block_size != 4096) { - printf("%s: unexpected block size %zu\n", name, map->block_size); - return NULL; - } - - map->region_count = strtoul(p, &p, 0); - map->regions = (int*) malloc(map->region_count * sizeof(int)); - map->total_blocks = 0; - - int i; - for (i = 0; i < map->region_count; ++i) { - map->regions[i] = strtoul(p, &p, 0); - map->total_blocks += map->regions[i]; - } - - return map; -} - -static FILE* mapwrite_cmd_pipe; - -static void progress_cb(long done, long total) { - if (total > 0) { - double frac = (double)done / total; - fprintf(mapwrite_cmd_pipe, "set_progress %f\n", frac); - fflush(mapwrite_cmd_pipe); - } -} - - - -bool MapWriter(const unsigned char* data, int dataLen, void* cookie) { - return write_with_map(data, dataLen, (MapState*) cookie, progress_cb) == dataLen; -} - -// package_extract_file(package_path, destination_path, map_path) -// or // package_extract_file(package_path, destination_path) // or // package_extract_file(package_path) @@ -533,33 +470,22 @@ bool MapWriter(const unsigned char* data, int dataLen, void* cookie) { // function (the char* returned is actually a FileContents*). Value* PackageExtractFileFn(const char* name, State* state, int argc, Expr* argv[]) { - if (argc < 1 || argc > 3) { - return ErrorAbort(state, "%s() expects 1 or 2 or 3 args, got %d", + if (argc < 1 || argc > 2) { + return ErrorAbort(state, "%s() expects 1 or 2 args, got %d", name, argc); } bool success = false; UpdaterInfo* ui = (UpdaterInfo*)(state->cookie); - mapwrite_cmd_pipe = ui->cmd_pipe; - if (argc >= 2) { - // The two-argument version extracts to a file; the three-arg - // version extracts to a file, skipping over regions in a - // don't care map. + if (argc == 2) { + // The two-argument version extracts to a file. ZipArchive* za = ((UpdaterInfo*)(state->cookie))->package_zip; char* zip_path; char* dest_path; - char* map_path = NULL; - DontCareMap* map = NULL; - if (argc == 2) { - if (ReadArgs(state, argv, 2, &zip_path, &dest_path) < 0) return NULL; - } else { - if (ReadArgs(state, argv, 3, &zip_path, &dest_path, &map_path) < 0) return NULL; - map = ReadDontCareMapFromZip(za, map_path); - if (map == NULL) goto done2; - } + if (ReadArgs(state, argv, 2, &zip_path, &dest_path) < 0) return NULL; const ZipEntry* entry = mzFindZipEntry(za, zip_path); if (entry == NULL) { @@ -573,26 +499,12 @@ Value* PackageExtractFileFn(const char* name, State* state, name, dest_path, strerror(errno)); goto done2; } - if (map) { - MapState state; - state.map = map; - state.cr = 0; - state.so_far = 0; - state.f = f; - success = mzProcessZipEntryContents(za, entry, MapWriter, &state); - } else { - success = mzExtractZipEntryToFile(za, entry, fileno(f)); - } + success = mzExtractZipEntryToFile(za, entry, fileno(f)); fclose(f); done2: free(zip_path); free(dest_path); - free(map_path); - if (map) { - free(map->regions); - free(map); - } return StringValue(strdup(success ? "t" : "")); } else { // The one-argument version returns the contents of the file @@ -1192,118 +1104,6 @@ Value* ApplyPatchSpaceFn(const char* name, State* state, return StringValue(strdup(CacheSizeCheck(bytes) ? "" : "t")); } -bool CheckMappedFileSha1(FILE* f, DontCareMap* map, uint8_t* intended_digest) { - MapState state; - - state.f = f; - state.so_far = 0; - state.cr = 0; - state.map = map; - - SHA_CTX ctx; - SHA_init(&ctx); - - unsigned char buffer[32173]; - size_t bytes_read; - - while ((bytes_read = read_with_map(buffer, sizeof(buffer), &state)) > 0) { - SHA_update(&ctx, buffer, bytes_read); - } - const uint8_t* digest = SHA_final(&ctx); - - return memcmp(digest, intended_digest, SHA_DIGEST_SIZE) == 0; -} - - -// syspatch(file, tgt_mapfile, tgt_sha1, init_mapfile, init_sha1, patch) - -Value* SysPatchFn(const char* name, State* state, int argc, Expr* argv[]) { - if (argc != 6) { - return ErrorAbort(state, "%s(): expected 6 args, got %d", name, argc); - } - - char* filename; - char* target_mapfilename; - char* target_sha1; - char* init_mapfilename; - char* init_sha1; - char* patch_filename; - uint8_t target_digest[SHA_DIGEST_SIZE]; - uint8_t init_digest[SHA_DIGEST_SIZE]; - - if (ReadArgs(state, argv, 6, &filename, - &target_mapfilename, &target_sha1, - &init_mapfilename, &init_sha1, &patch_filename) < 0) { - return NULL; - } - - UpdaterInfo* ui = (UpdaterInfo*)(state->cookie); - mapwrite_cmd_pipe = ui->cmd_pipe; - - if (ParseSha1(target_sha1, target_digest) != 0) { - printf("%s(): failed to parse '%s' as target SHA-1", name, target_sha1); - memset(target_digest, 0, SHA_DIGEST_SIZE); - } - if (ParseSha1(init_sha1, init_digest) != 0) { - printf("%s(): failed to parse '%s' as init SHA-1", name, init_sha1); - memset(init_digest, 0, SHA_DIGEST_SIZE); - } - - ZipArchive* za = ((UpdaterInfo*)(state->cookie))->package_zip; - FILE* src = fopen(filename, "r"); - - DontCareMap* init_map = ReadDontCareMapFromZip(za, init_mapfilename); - if (init_map == NULL) return ErrorAbort(state, "%s(): failed to read init map\n", name); - DontCareMap* target_map = ReadDontCareMapFromZip(za, target_mapfilename); - if (target_map == NULL) return ErrorAbort(state, "%s(): failed to read target map\n", name); - - if (CheckMappedFileSha1(src, init_map, init_digest)) { - // If the partition contents match the init_digest, then we need to apply the patch. - - rewind(src); - - const ZipEntry* entry = mzFindZipEntry(za, patch_filename); - if (entry == NULL) { - return ErrorAbort(state, "%s(): no %s in package\n", name, patch_filename); - } - - unsigned char* patch_data; - size_t patch_len; - if (!mzGetStoredEntry(za, entry, &patch_data, &patch_len)) { - return ErrorAbort(state, "%s(): failed to get %s entry\n", name, patch_filename); - } - - FILE* tgt = fopen(filename, "r+"); - - int ret = syspatch(src, init_map, patch_data, patch_len, tgt, target_map, progress_cb); - - fclose(src); - fclose(tgt); - - if (ret != 0) { - return ErrorAbort(state, "%s(): patching failed\n", name); - } - } else { - rewind(src); - if (CheckMappedFileSha1(src, target_map, target_digest)) { - // If the partition contents match the target already, we - // don't need to do anything. - printf("%s: output is already target\n", name); - } else { - return ErrorAbort(state, "%s(): %s in unknown state\n", name, filename); - } - } - - done: - free(target_sha1); - free(target_mapfilename); - free(init_sha1); - free(init_mapfilename); - free(patch_filename); - return StringValue(filename); - -} - // apply_patch(file, size, init_sha1, tgt_sha1, patch) Value* ApplyPatchFn(const char* name, State* state, int argc, Expr* argv[]) { @@ -1738,7 +1538,6 @@ void RegisterInstallFunctions() { RegisterFunction("apply_patch_space", ApplyPatchSpaceFn); RegisterFunction("wipe_block_device", WipeBlockDeviceFn); - RegisterFunction("syspatch", SysPatchFn); RegisterFunction("read_file", ReadFileFn); RegisterFunction("sha1_check", Sha1CheckFn); From 52ae67d659e92bdd32c7c2a6b4d09dfe94e987fe Mon Sep 17 00:00:00 2001 From: Doug Zongker Date: Mon, 8 Sep 2014 12:22:09 -0700 Subject: [PATCH 04/13] support for version 2 of block image diffs In version 2 of block image diffs, we support a new command to load data from the image and store it in the "stash table" and then subsequently use entries in the stash table to fill in missing bits of source data we're not allowed to read when doing move/bsdiff/imgdiff commands. This leads to smaller update packages because we can break cycles in the ordering of how pieces are updated by storing data away and using it later, rather than not using the data as input to the patch system at all. This comes at the cost of the RAM or scratch disk needed to store the data. The implementation is backwards compatible; it can still handle the existing version 1 of the transfer file format. Change-Id: I7fafe741d86b92d82d46feb2939ecf5a3890dc64 --- updater/blockimg.c | 256 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 210 insertions(+), 46 deletions(-) diff --git a/updater/blockimg.c b/updater/blockimg.c index c3319c97..30268931 100644 --- a/updater/blockimg.c +++ b/updater/blockimg.c @@ -61,7 +61,7 @@ static RangeSet* parse_range(char* text) { RangeSet* out = malloc(sizeof(RangeSet) + num * sizeof(int)); if (out == NULL) { - fprintf(stderr, "failed to allocate range of %lu bytes\n", + fprintf(stderr, "failed to allocate range of %zu bytes\n", sizeof(RangeSet) + num * sizeof(int)); exit(1); } @@ -245,6 +245,133 @@ static void* unzip_new_data(void* cookie) { return NULL; } +// Do a source/target load for move/bsdiff/imgdiff in version 1. +// 'wordsave' is the save_ptr of a strtok_r()-in-progress. We expect +// to parse the remainder of the string as: +// +// +// +// The source range is loaded into the provided buffer, reallocating +// it to make it larger if necessary. The target ranges are returned +// in *tgt, if tgt is non-NULL. + +static void LoadSrcTgtVersion1(char* wordsave, RangeSet** tgt, int* src_blocks, + uint8_t** buffer, size_t* buffer_alloc, int fd) { + char* word; + + word = strtok_r(NULL, " ", &wordsave); + RangeSet* src = parse_range(word); + + if (tgt != NULL) { + word = strtok_r(NULL, " ", &wordsave); + *tgt = parse_range(word); + } + + allocate(src->size * BLOCKSIZE, buffer, buffer_alloc); + size_t p = 0; + int i; + for (i = 0; i < src->count; ++i) { + check_lseek(fd, (off64_t)src->pos[i*2] * BLOCKSIZE, SEEK_SET); + size_t sz = (src->pos[i*2+1] - src->pos[i*2]) * BLOCKSIZE; + readblock(fd, *buffer+p, sz); + p += sz; + } + + *src_blocks = src->size; + free(src); +} + +static void MoveRange(uint8_t* dest, RangeSet* locs, const uint8_t* source) { + // source contains packed data, which we want to move to the + // locations given in *locs in the dest buffer. source and dest + // may be the same buffer. + + int start = locs->size; + int i; + for (i = locs->count-1; i >= 0; --i) { + int blocks = locs->pos[i*2+1] - locs->pos[i*2]; + start -= blocks; + memmove(dest + (locs->pos[i*2] * BLOCKSIZE), source + (start * BLOCKSIZE), + blocks * BLOCKSIZE); + } +} + +// Do a source/target load for move/bsdiff/imgdiff in version 2. +// 'wordsave' is the save_ptr of a strtok_r()-in-progress. We expect +// to parse the remainder of the string as one of: +// +// +// (loads data from source image only) +// +// - <[stash_id:stash_range] ...> +// (loads data from stashes only) +// +// <[stash_id:stash_range] ...> +// (loads data from both source image and stashes) +// +// On return, buffer is filled with the loaded source data (rearranged +// and combined with stashed data as necessary). buffer may be +// reallocated if needed to accommodate the source data. *tgt is the +// target RangeSet. Any stashes required are taken from stash_table +// and free()'d after being used. + +static void LoadSrcTgtVersion2(char* wordsave, RangeSet** tgt, int* src_blocks, + uint8_t** buffer, size_t* buffer_alloc, int fd, + uint8_t** stash_table) { + char* word; + + if (tgt != NULL) { + word = strtok_r(NULL, " ", &wordsave); + *tgt = parse_range(word); + } + + word = strtok_r(NULL, " ", &wordsave); + *src_blocks = strtol(word, NULL, 0); + + allocate(*src_blocks * BLOCKSIZE, buffer, buffer_alloc); + + word = strtok_r(NULL, " ", &wordsave); + if (word[0] == '-' && word[1] == '\0') { + // no source ranges, only stashes + } else { + RangeSet* src = parse_range(word); + + size_t p = 0; + int i; + for (i = 0; i < src->count; ++i) { + check_lseek(fd, (off64_t)src->pos[i*2] * BLOCKSIZE, SEEK_SET); + size_t sz = (src->pos[i*2+1] - src->pos[i*2]) * BLOCKSIZE; + readblock(fd, *buffer+p, sz); + p += sz; + } + free(src); + + word = strtok_r(NULL, " ", &wordsave); + if (word == NULL) { + // no stashes, only source range + return; + } + + RangeSet* locs = parse_range(word); + MoveRange(*buffer, locs, *buffer); + } + + while ((word = strtok_r(NULL, " ", &wordsave)) != NULL) { + // Each word is a an index into the stash table, a colon, and + // then a rangeset describing where in the source block that + // stashed data should go. + char* colonsave = NULL; + char* colon = strtok_r(word, ":", &colonsave); + int stash_id = strtol(colon, NULL, 0); + colon = strtok_r(NULL, ":", &colonsave); + RangeSet* locs = parse_range(colon); + MoveRange(*buffer, locs, stash_table[stash_id]); + free(stash_table[stash_id]); + stash_table[stash_id] = NULL; + free(locs); + } +} + // args: // - block device (or file) to modify in-place // - transfer list (blob) @@ -311,23 +438,33 @@ Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[] // new [rangeset] // - fill the blocks with data read from the new_data file // - // bsdiff patchstart patchlen [src rangeset] [tgt rangeset] - // imgdiff patchstart patchlen [src rangeset] [tgt rangeset] - // - read the source blocks, apply a patch, write result to - // target blocks. bsdiff or imgdiff specifies the type of - // patch. - // - // move [src rangeset] [tgt rangeset] - // - copy data from source blocks to target blocks (no patch - // needed; rangesets are the same size) - // // erase [rangeset] // - mark the given blocks as empty // + // move <...> + // bsdiff <...> + // imgdiff <...> + // - read the source blocks, apply a patch (or not in the + // case of move), write result to target blocks. bsdiff or + // imgdiff specifies the type of patch; move means no patch + // at all. + // + // The format of <...> differs between versions 1 and 2; + // see the LoadSrcTgtVersion{1,2}() functions for a + // description of what's expected. + // + // stash + // - (version 2 only) load the given source range and stash + // the data in the given slot of the stash table. + // // The creator of the transfer list will guarantee that no block // is read (ie, used as the source for a patch or move) after it // has been written. // + // In version 2, the creator will guarantee that a given stash is + // loaded (with a stash command) before it's used in a + // move/bsdiff/imgdiff command. + // // Within one command the source and target ranges may overlap so // in general we need to read the entire source into memory before // writing anything to the target blocks. @@ -379,12 +516,18 @@ Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[] line = strtok_r(transfer_list, "\n", &linesave); + int version; // first line in transfer list is the version number; currently // there's only version 1. - if (strcmp(line, "1") != 0) { + if (strcmp(line, "1") == 0) { + version = 1; + } else if (strcmp(line, "2") == 0) { + version = 2; + } else { ErrorAbort(state, "unexpected transfer list version [%s]\n", line); goto done; } + printf("blockimg version is %d\n", version); // second line in transfer list is the total number of blocks we // expect to write. @@ -394,33 +537,49 @@ Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[] if (total_blocks == 0) ++total_blocks; int blocks_so_far = 0; + uint8_t** stash_table = NULL; + if (version >= 2) { + // Next line is how many stash entries are needed simultaneously. + line = strtok_r(NULL, "\n", &linesave); + int stash_entries = strtol(line, NULL, 0); + + stash_table = (uint8_t**) calloc(stash_entries, sizeof(uint8_t*)); + if (stash_table == NULL) { + fprintf(stderr, "failed to allocate %d-entry stash table\n", stash_entries); + exit(1); + } + + // Next line is the maximum number of blocks that will be + // stashed simultaneously. This could be used to verify that + // enough memory or scratch disk space is available. + line = strtok_r(NULL, "\n", &linesave); + int stash_max_blocks = strtol(line, NULL, 0); + } + uint8_t* buffer = NULL; size_t buffer_alloc = 0; // third and subsequent lines are all individual transfer commands. for (line = strtok_r(NULL, "\n", &linesave); line; line = strtok_r(NULL, "\n", &linesave)) { + char* style; style = strtok_r(line, " ", &wordsave); if (strcmp("move", style) == 0) { - word = strtok_r(NULL, " ", &wordsave); - RangeSet* src = parse_range(word); - word = strtok_r(NULL, " ", &wordsave); - RangeSet* tgt = parse_range(word); - - printf(" moving %d blocks\n", src->size); - - allocate(src->size * BLOCKSIZE, &buffer, &buffer_alloc); - size_t p = 0; - for (i = 0; i < src->count; ++i) { - check_lseek(fd, (off64_t)src->pos[i*2] * BLOCKSIZE, SEEK_SET); - size_t sz = (src->pos[i*2+1] - src->pos[i*2]) * BLOCKSIZE; - readblock(fd, buffer+p, sz); - p += sz; + RangeSet* tgt; + int src_blocks; + if (version == 1) { + LoadSrcTgtVersion1(wordsave, &tgt, &src_blocks, + &buffer, &buffer_alloc, fd); + } else if (version == 2) { + LoadSrcTgtVersion2(wordsave, &tgt, &src_blocks, + &buffer, &buffer_alloc, fd, stash_table); } - p = 0; + printf(" moving %d blocks\n", src_blocks); + + size_t p = 0; for (i = 0; i < tgt->count; ++i) { check_lseek(fd, (off64_t)tgt->pos[i*2] * BLOCKSIZE, SEEK_SET); size_t sz = (tgt->pos[i*2+1] - tgt->pos[i*2]) * BLOCKSIZE; @@ -432,9 +591,20 @@ Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[] fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); fflush(cmd_pipe); - free(src); free(tgt); + } else if (strcmp("stash", style) == 0) { + word = strtok_r(NULL, " ", &wordsave); + int stash_id = strtol(word, NULL, 0); + int src_blocks; + size_t stash_alloc = 0; + + // Even though the "stash" style only appears in version + // 2, the version 1 source loader happens to do exactly + // what we want to read data into the stash_table. + LoadSrcTgtVersion1(wordsave, NULL, &src_blocks, + stash_table + stash_id, &stash_alloc, fd); + } else if (strcmp("zero", style) == 0 || (DEBUG_ERASE && strcmp("erase", style) == 0)) { word = strtok_r(NULL, " ", &wordsave); @@ -493,23 +663,18 @@ Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[] word = strtok_r(NULL, " ", &wordsave); size_t patch_len = strtoul(word, NULL, 0); - word = strtok_r(NULL, " ", &wordsave); - RangeSet* src = parse_range(word); - word = strtok_r(NULL, " ", &wordsave); - RangeSet* tgt = parse_range(word); - - printf(" patching %d blocks to %d\n", src->size, tgt->size); - - // Read the source into memory. - allocate(src->size * BLOCKSIZE, &buffer, &buffer_alloc); - size_t p = 0; - for (i = 0; i < src->count; ++i) { - check_lseek(fd, (off64_t)src->pos[i*2] * BLOCKSIZE, SEEK_SET); - size_t sz = (src->pos[i*2+1] - src->pos[i*2]) * BLOCKSIZE; - readblock(fd, buffer+p, sz); - p += sz; + RangeSet* tgt; + int src_blocks; + if (version == 1) { + LoadSrcTgtVersion1(wordsave, &tgt, &src_blocks, + &buffer, &buffer_alloc, fd); + } else if (version == 2) { + LoadSrcTgtVersion2(wordsave, &tgt, &src_blocks, + &buffer, &buffer_alloc, fd, stash_table); } + printf(" patching %d blocks to %d\n", src_blocks, tgt->size); + Value patch_value; patch_value.type = VAL_BLOB; patch_value.size = patch_len; @@ -523,11 +688,11 @@ Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[] check_lseek(fd, (off64_t)tgt->pos[0] * BLOCKSIZE, SEEK_SET); if (style[0] == 'i') { // imgdiff - ApplyImagePatch(buffer, src->size * BLOCKSIZE, + ApplyImagePatch(buffer, src_blocks * BLOCKSIZE, &patch_value, &RangeSinkWrite, &rss, NULL, NULL); } else { - ApplyBSDiffPatch(buffer, src->size * BLOCKSIZE, + ApplyBSDiffPatch(buffer, src_blocks * BLOCKSIZE, &patch_value, 0, &RangeSinkWrite, &rss, NULL); } @@ -541,7 +706,6 @@ Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[] fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); fflush(cmd_pipe); - free(src); free(tgt); } else if (!DEBUG_ERASE && strcmp("erase", style) == 0) { struct stat st; From 1d30c2fe19fdbdfd6e5f52102247cf01b87e586e Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Tue, 16 Sep 2014 14:53:39 -0700 Subject: [PATCH 05/13] Use the correct fuse_init_out structure size. Kernel 2.6.16 is the first stable kernel with struct fuse_init_out defined (fuse version 7.6). The structure is the same from 7.6 through 7.22. Beginning with 7.23, the structure increased in size and added new parameters. If the kernel only works on minor revs older than or equal to 22, then use the older structure size since this code only uses the 7.22 version of the structure. Change-Id: I00d7530e01e6b4718dcd04ad2484959d12ef4a65 --- fuse_sideload.c | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/fuse_sideload.c b/fuse_sideload.c index ab91defb..4e11e01e 100644 --- a/fuse_sideload.c +++ b/fuse_sideload.c @@ -53,6 +53,7 @@ #include #include #include +#include #include #include #include @@ -117,15 +118,40 @@ static void fuse_reply(struct fuse_data* fd, __u64 unique, const void *data, siz static int handle_init(void* data, struct fuse_data* fd, const struct fuse_in_header* hdr) { const struct fuse_init_in* req = data; struct fuse_init_out out; + size_t fuse_struct_size; + + + /* Kernel 2.6.16 is the first stable kernel with struct fuse_init_out + * defined (fuse version 7.6). The structure is the same from 7.6 through + * 7.22. Beginning with 7.23, the structure increased in size and added + * new parameters. + */ + if (req->major != FUSE_KERNEL_VERSION || req->minor < 6) { + printf("Fuse kernel version mismatch: Kernel version %d.%d, Expected at least %d.6", + req->major, req->minor, FUSE_KERNEL_VERSION); + return -1; + } + + out.minor = MIN(req->minor, FUSE_KERNEL_MINOR_VERSION); + fuse_struct_size = sizeof(out); +#if defined(FUSE_COMPAT_22_INIT_OUT_SIZE) + /* FUSE_KERNEL_VERSION >= 23. */ + + /* If the kernel only works on minor revs older than or equal to 22, + * then use the older structure size since this code only uses the 7.22 + * version of the structure. */ + if (req->minor <= 22) { + fuse_struct_size = FUSE_COMPAT_22_INIT_OUT_SIZE; + } +#endif out.major = FUSE_KERNEL_VERSION; - out.minor = FUSE_KERNEL_MINOR_VERSION; out.max_readahead = req->max_readahead; out.flags = 0; out.max_background = 32; out.congestion_threshold = 32; out.max_write = 4096; - fuse_reply(fd, hdr->unique, &out, sizeof(out)); + fuse_reply(fd, hdr->unique, &out, fuse_struct_size); return NO_STATUS; } From b5b43043fa71f9cb620ddd02ec2bc98eced5a6ce Mon Sep 17 00:00:00 2001 From: Badhri Jagan Sridharan Date: Mon, 27 Oct 2014 18:32:35 -0700 Subject: [PATCH 06/13] adbd: Support for new f_fs descriptor format The patch "[RFC] usb: gadget: f_fs: Add flags to descriptors block" marks the current usb_functionfs_descs_head format deprecated and introduces support for sending SuperSpeed descriptors. This CL makes adbd to send Descriptors in the new format. Adbd would fall back to the old format, if kernel is not able to recognize the new format. This is done to prevent adbd from breaking in the older versions of the kernel. Bug: 17394972 Change-Id: I1acf684ef8a4dcc612ac20b5abe1e27b43901031 Signed-off-by: Badhri Jagan Sridharan --- minadbd/usb_linux_client.c | 161 ++++++++++++++++++++++++------------- 1 file changed, 107 insertions(+), 54 deletions(-) diff --git a/minadbd/usb_linux_client.c b/minadbd/usb_linux_client.c index 29bab155..e7d3c485 100644 --- a/minadbd/usb_linux_client.c +++ b/minadbd/usb_linux_client.c @@ -52,7 +52,35 @@ struct usb_handle int bulk_in; /* "in" from the host's perspective => sink for adbd */ }; -static const struct { +struct func_desc { + struct usb_interface_descriptor intf; + struct usb_endpoint_descriptor_no_audio source; + struct usb_endpoint_descriptor_no_audio sink; +} __attribute__((packed)); + +struct desc_v1 { + struct usb_functionfs_descs_head_v1 { + __le32 magic; + __le32 length; + __le32 fs_count; + __le32 hs_count; + } __attribute__((packed)) header; + struct func_desc fs_descs, hs_descs; +} __attribute__((packed)); + +struct desc_v2 { + struct usb_functionfs_descs_head_v2 { + __le32 magic; + __le32 length; + __le32 flags; + __le32 fs_count; + __le32 hs_count; + __le32 ss_count; + } __attribute__((packed)) header; + struct func_desc fs_descs, hs_descs; +} __attribute__((packed)); + +/*static const struct { struct usb_functionfs_descs_head header; struct { struct usb_interface_descriptor intf; @@ -66,57 +94,60 @@ static const struct { .fs_count = 3, .hs_count = 3, }, - .fs_descs = { - .intf = { - .bLength = sizeof(descriptors.fs_descs.intf), - .bDescriptorType = USB_DT_INTERFACE, - .bInterfaceNumber = 0, - .bNumEndpoints = 2, - .bInterfaceClass = ADB_CLASS, - .bInterfaceSubClass = ADB_SUBCLASS, - .bInterfaceProtocol = ADB_PROTOCOL, - .iInterface = 1, /* first string from the provided table */ - }, - .source = { - .bLength = sizeof(descriptors.fs_descs.source), - .bDescriptorType = USB_DT_ENDPOINT, - .bEndpointAddress = 1 | USB_DIR_OUT, - .bmAttributes = USB_ENDPOINT_XFER_BULK, - .wMaxPacketSize = MAX_PACKET_SIZE_FS, - }, - .sink = { - .bLength = sizeof(descriptors.fs_descs.sink), - .bDescriptorType = USB_DT_ENDPOINT, - .bEndpointAddress = 2 | USB_DIR_IN, - .bmAttributes = USB_ENDPOINT_XFER_BULK, - .wMaxPacketSize = MAX_PACKET_SIZE_FS, - }, + +*/ + +struct func_desc fs_descriptors = { + .intf = { + .bLength = sizeof(fs_descriptors.intf), + .bDescriptorType = USB_DT_INTERFACE, + .bInterfaceNumber = 0, + .bNumEndpoints = 2, + .bInterfaceClass = ADB_CLASS, + .bInterfaceSubClass = ADB_SUBCLASS, + .bInterfaceProtocol = ADB_PROTOCOL, + .iInterface = 1, /* first string from the provided table */ }, - .hs_descs = { - .intf = { - .bLength = sizeof(descriptors.hs_descs.intf), - .bDescriptorType = USB_DT_INTERFACE, - .bInterfaceNumber = 0, - .bNumEndpoints = 2, - .bInterfaceClass = ADB_CLASS, - .bInterfaceSubClass = ADB_SUBCLASS, - .bInterfaceProtocol = ADB_PROTOCOL, - .iInterface = 1, /* first string from the provided table */ - }, - .source = { - .bLength = sizeof(descriptors.hs_descs.source), - .bDescriptorType = USB_DT_ENDPOINT, - .bEndpointAddress = 1 | USB_DIR_OUT, - .bmAttributes = USB_ENDPOINT_XFER_BULK, - .wMaxPacketSize = MAX_PACKET_SIZE_HS, - }, - .sink = { - .bLength = sizeof(descriptors.hs_descs.sink), - .bDescriptorType = USB_DT_ENDPOINT, - .bEndpointAddress = 2 | USB_DIR_IN, - .bmAttributes = USB_ENDPOINT_XFER_BULK, - .wMaxPacketSize = MAX_PACKET_SIZE_HS, - }, + .source = { + .bLength = sizeof(fs_descriptors.source), + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = 1 | USB_DIR_OUT, + .bmAttributes = USB_ENDPOINT_XFER_BULK, + .wMaxPacketSize = MAX_PACKET_SIZE_FS, + }, + .sink = { + .bLength = sizeof(fs_descriptors.sink), + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = 2 | USB_DIR_IN, + .bmAttributes = USB_ENDPOINT_XFER_BULK, + .wMaxPacketSize = MAX_PACKET_SIZE_FS, + }, +}; + +struct func_desc hs_descriptors = { + .intf = { + .bLength = sizeof(hs_descriptors.intf), + .bDescriptorType = USB_DT_INTERFACE, + .bInterfaceNumber = 0, + .bNumEndpoints = 2, + .bInterfaceClass = ADB_CLASS, + .bInterfaceSubClass = ADB_SUBCLASS, + .bInterfaceProtocol = ADB_PROTOCOL, + .iInterface = 1, /* first string from the provided table */ + }, + .source = { + .bLength = sizeof(hs_descriptors.source), + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = 1 | USB_DIR_OUT, + .bmAttributes = USB_ENDPOINT_XFER_BULK, + .wMaxPacketSize = MAX_PACKET_SIZE_HS, + }, + .sink = { + .bLength = sizeof(hs_descriptors.sink), + .bDescriptorType = USB_DT_ENDPOINT, + .bEndpointAddress = 2 | USB_DIR_IN, + .bmAttributes = USB_ENDPOINT_XFER_BULK, + .wMaxPacketSize = MAX_PACKET_SIZE_HS, }, }; @@ -267,6 +298,18 @@ static void usb_adb_init() static void init_functionfs(struct usb_handle *h) { ssize_t ret; + struct desc_v1 v1_descriptor; + struct desc_v2 v2_descriptor; + + v2_descriptor.header.magic = cpu_to_le32(FUNCTIONFS_DESCRIPTORS_MAGIC_V2); + v2_descriptor.header.length = cpu_to_le32(sizeof(v2_descriptor)); + v2_descriptor.header.flags = FUNCTIONFS_HAS_FS_DESC | FUNCTIONFS_HAS_HS_DESC; + v2_descriptor.header.fs_count = 3; + v2_descriptor.header.hs_count = 3; + v2_descriptor.header.ss_count = 0; + v2_descriptor.fs_descs = fs_descriptors; + v2_descriptor.hs_descs = hs_descriptors; + D("OPENING %s\n", USB_FFS_ADB_EP0); h->control = adb_open(USB_FFS_ADB_EP0, O_RDWR); @@ -275,10 +318,20 @@ static void init_functionfs(struct usb_handle *h) goto err; } - ret = adb_write(h->control, &descriptors, sizeof(descriptors)); + ret = adb_write(h->control, &v2_descriptor, sizeof(v2_descriptor)); if (ret < 0) { - D("[ %s: write descriptors failed: errno=%d ]\n", USB_FFS_ADB_EP0, errno); - goto err; + v1_descriptor.header.magic = cpu_to_le32(FUNCTIONFS_DESCRIPTORS_MAGIC); + v1_descriptor.header.length = cpu_to_le32(sizeof(v1_descriptor)); + v1_descriptor.header.fs_count = 3; + v1_descriptor.header.hs_count = 3; + v1_descriptor.fs_descs = fs_descriptors; + v1_descriptor.hs_descs = hs_descriptors; + D("[ %s: Switching to V1_descriptor format errno=%d ]\n", USB_FFS_ADB_EP0, errno); + ret = adb_write(h->control, &v1_descriptor, sizeof(v1_descriptor)); + if (ret < 0) { + D("[ %s: write descriptors failed: errno=%d ]\n", USB_FFS_ADB_EP0, errno); + goto err; + } } ret = adb_write(h->control, &strings, sizeof(strings)); From acf47db238b2f4fff01969210945eec2528de9b7 Mon Sep 17 00:00:00 2001 From: Michael Runge Date: Fri, 21 Nov 2014 00:12:28 -0800 Subject: [PATCH 07/13] Add support for tune2fs file operations This allows tune2fs to be executed from within OTA scripts, allowing for file system modifications without formatting the partition Bug: 18430740 Change-Id: I0c2e05b5ef4a81ecea043e9b7b99b545d18fe5e6 --- updater/Android.mk | 10 ++++++++++ updater/install.c | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/updater/Android.mk b/updater/Android.mk index a3a900a8..11e7bb80 100644 --- a/updater/Android.mk +++ b/updater/Android.mk @@ -34,6 +34,16 @@ LOCAL_STATIC_LIBRARIES += libapplypatch libedify libmtdutils libminzip libz LOCAL_STATIC_LIBRARIES += libmincrypt libbz LOCAL_STATIC_LIBRARIES += libcutils liblog libstdc++ libc LOCAL_STATIC_LIBRARIES += libselinux +tune2fs_static_libraries := \ + libext2_com_err \ + libext2_blkid \ + libext2_quota \ + libext2_uuid_static \ + libext2_e2p \ + libext2fs +LOCAL_STATIC_LIBRARIES += libtune2fs $(tune2fs_static_libraries) + +LOCAL_C_INCLUDES += external/e2fsprogs/misc LOCAL_C_INCLUDES += $(LOCAL_PATH)/.. # Each library in TARGET_RECOVERY_UPDATER_LIBS should have a function diff --git a/updater/install.c b/updater/install.c index ff7de479..2b2ffb0c 100644 --- a/updater/install.c +++ b/updater/install.c @@ -46,6 +46,7 @@ #include "mtdutils/mtdutils.h" #include "updater.h" #include "install.h" +#include "tune2fs.h" #ifdef USE_EXT4 #include "make_ext4fs.h" @@ -1539,6 +1540,37 @@ Value* EnableRebootFn(const char* name, State* state, int argc, Expr* argv[]) { return StringValue(strdup("t")); } +Value* Tune2FsFn(const char* name, State* state, int argc, Expr* argv[]) { + if (argc == 0) { + return ErrorAbort(state, "%s() expects args, got %d", name, argc); + } + + char** args = ReadVarArgs(state, argc, argv); + if (args == NULL) { + return ErrorAbort(state, "%s() could not read args", name); + } + + int i; + char** args2 = malloc(sizeof(char*) * (argc+1)); + // Tune2fs expects the program name as its args[0] + args2[0] = strdup(name); + for (i = 0; i < argc; ++i) { + args2[i + 1] = args[i]; + } + int result = tune2fs_main(argc + 1, args2); + for (i = 0; i < argc; ++i) { + free(args[i]); + } + free(args); + + free(args2[0]); + free(args2); + if (result != 0) { + return ErrorAbort(state, "%s() returned error code %d", name, result); + } + return StringValue(strdup("t")); +} + void RegisterInstallFunctions() { RegisterFunction("mount", MountFn); RegisterFunction("is_mounted", IsMountedFn); @@ -1589,4 +1621,5 @@ void RegisterInstallFunctions() { RegisterFunction("set_stage", SetStageFn); RegisterFunction("enable_reboot", EnableRebootFn); + RegisterFunction("tune2fs", Tune2FsFn); } From 90221205a3e58f2a198faa838088dc7bc7c9c752 Mon Sep 17 00:00:00 2001 From: Sami Tolvanen Date: Tue, 9 Dec 2014 16:39:47 +0000 Subject: [PATCH 08/13] Support resuming block based OTAs Add support for transfer list version 3, which allows us to verify the status of each command and resume an interrupted block based OTA update. Notes on the changes: - Move the previous BlockImageUpdateFn to a shorter and reusable PerformBlockImageUpdate, which can be used also in BlockImageVerifyFn for verification. - Split individual transfer list commands into separate functions with unified parameters for clarity, and use a hash table to locate them during execution. - Move common block reading and writing to ReadBlocks and WriteBlocks to reduce code duplication, and rename the readblock and writeblock to less confusing read_all and write_all. The coding style of the new functions follows the existing style in the updater/edify code. Needs matching changes from Ia5c56379f570047f10f0aa7373a1025439495c98 Bug: 18262110 Change-Id: I1e752464134aeb2d396946348e6041acabe13942 --- updater/blockimg.c | 1842 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 1478 insertions(+), 364 deletions(-) diff --git a/updater/blockimg.c b/updater/blockimg.c index 30268931..a0f81e93 100644 --- a/updater/blockimg.c +++ b/updater/blockimg.c @@ -16,6 +16,7 @@ #include #include +#include #include #include #include @@ -23,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -32,7 +34,7 @@ #include "applypatch/applypatch.h" #include "edify/expr.h" #include "mincrypt/sha.h" -#include "minzip/DirUtil.h" +#include "minzip/Hash.h" #include "updater.h" #define BLOCKSIZE 4096 @@ -46,6 +48,10 @@ #define BLKDISCARD _IO(0x12,119) #endif +#define STASH_DIRECTORY_BASE "/cache/recovery" +#define STASH_DIRECTORY_MODE 0700 +#define STASH_FILE_MODE 0600 + char* PrintSha1(const uint8_t* digest); typedef struct { @@ -80,44 +86,77 @@ static RangeSet* parse_range(char* text) { return out; } -static void readblock(int fd, uint8_t* data, size_t size) { +static int range_overlaps(RangeSet* r1, RangeSet* r2) { + int i, j, r1_0, r1_1, r2_0, r2_1; + + if (!r1 || !r2) { + return 0; + } + + for (i = 0; i < r1->count; ++i) { + r1_0 = r1->pos[i * 2]; + r1_1 = r1->pos[i * 2 + 1]; + + for (j = 0; j < r2->count; ++j) { + r2_0 = r2->pos[j * 2]; + r2_1 = r2->pos[j * 2 + 1]; + + if (!(r2_0 > r1_1 || r1_0 > r2_1)) { + return 1; + } + } + } + + return 0; +} + +static int read_all(int fd, uint8_t* data, size_t size) { size_t so_far = 0; while (so_far < size) { ssize_t r = read(fd, data+so_far, size-so_far); if (r < 0 && errno != EINTR) { fprintf(stderr, "read failed: %s\n", strerror(errno)); - return; + return -1; } else { so_far += r; } } + return 0; } -static void writeblock(int fd, const uint8_t* data, size_t size) { +static int write_all(int fd, const uint8_t* data, size_t size) { size_t written = 0; while (written < size) { ssize_t w = write(fd, data+written, size-written); if (w < 0 && errno != EINTR) { fprintf(stderr, "write failed: %s\n", strerror(errno)); - return; + return -1; } else { written += w; } } + + if (fsync(fd) == -1) { + fprintf(stderr, "fsync failed: %s\n", strerror(errno)); + return -1; + } + + return 0; } -static void check_lseek(int fd, off64_t offset, int whence) { +static int check_lseek(int fd, off64_t offset, int whence) { while (true) { off64_t ret = lseek64(fd, offset, whence); if (ret < 0) { if (errno != EINTR) { fprintf(stderr, "lseek64 failed: %s\n", strerror(errno)); - exit(1); + return -1; } } else { break; } } + return 0; } static void allocate(size_t size, uint8_t** buffer, size_t* buffer_alloc) { @@ -146,14 +185,21 @@ static ssize_t RangeSinkWrite(const uint8_t* data, ssize_t size, void* token) { if (rss->p_remain <= 0) { fprintf(stderr, "range sink write overrun"); - exit(1); + return 0; } ssize_t written = 0; while (size > 0) { size_t write_now = size; - if (rss->p_remain < write_now) write_now = rss->p_remain; - writeblock(rss->fd, data, write_now); + + if (rss->p_remain < write_now) { + write_now = rss->p_remain; + } + + if (write_all(rss->fd, data, write_now) == -1) { + break; + } + data += write_now; size -= write_now; @@ -164,12 +210,17 @@ static ssize_t RangeSinkWrite(const uint8_t* data, ssize_t size, void* token) { // move to the next block ++rss->p_block; if (rss->p_block < rss->tgt->count) { - rss->p_remain = (rss->tgt->pos[rss->p_block*2+1] - rss->tgt->pos[rss->p_block*2]) * BLOCKSIZE; - check_lseek(rss->fd, (off64_t)rss->tgt->pos[rss->p_block*2] * BLOCKSIZE, SEEK_SET); + rss->p_remain = (rss->tgt->pos[rss->p_block * 2 + 1] - + rss->tgt->pos[rss->p_block * 2]) * BLOCKSIZE; + + if (check_lseek(rss->fd, (off64_t)rss->tgt->pos[rss->p_block*2] * BLOCKSIZE, + SEEK_SET) == -1) { + break; + } } else { // we can't write any more; return how many bytes have // been written so far. - return written; + break; } } } @@ -245,6 +296,58 @@ static void* unzip_new_data(void* cookie) { return NULL; } +static int ReadBlocks(RangeSet* src, uint8_t* buffer, int fd) { + int i; + size_t p = 0; + size_t size; + + if (!src || !buffer) { + return -1; + } + + for (i = 0; i < src->count; ++i) { + if (check_lseek(fd, (off64_t) src->pos[i * 2] * BLOCKSIZE, SEEK_SET) == -1) { + return -1; + } + + size = (src->pos[i * 2 + 1] - src->pos[i * 2]) * BLOCKSIZE; + + if (read_all(fd, buffer + p, size) == -1) { + return -1; + } + + p += size; + } + + return 0; +} + +static int WriteBlocks(RangeSet* tgt, uint8_t* buffer, int fd) { + int i; + size_t p = 0; + size_t size; + + if (!tgt || !buffer) { + return -1; + } + + for (i = 0; i < tgt->count; ++i) { + if (check_lseek(fd, (off64_t) tgt->pos[i * 2] * BLOCKSIZE, SEEK_SET) == -1) { + return -1; + } + + size = (tgt->pos[i * 2 + 1] - tgt->pos[i * 2]) * BLOCKSIZE; + + if (write_all(fd, buffer + p, size) == -1) { + return -1; + } + + p += size; + } + + return 0; +} + // Do a source/target load for move/bsdiff/imgdiff in version 1. // 'wordsave' is the save_ptr of a strtok_r()-in-progress. We expect // to parse the remainder of the string as: @@ -255,30 +358,489 @@ static void* unzip_new_data(void* cookie) { // it to make it larger if necessary. The target ranges are returned // in *tgt, if tgt is non-NULL. -static void LoadSrcTgtVersion1(char* wordsave, RangeSet** tgt, int* src_blocks, +static int LoadSrcTgtVersion1(char** wordsave, RangeSet** tgt, int* src_blocks, uint8_t** buffer, size_t* buffer_alloc, int fd) { char* word; + int rc; - word = strtok_r(NULL, " ", &wordsave); + word = strtok_r(NULL, " ", wordsave); RangeSet* src = parse_range(word); if (tgt != NULL) { - word = strtok_r(NULL, " ", &wordsave); + word = strtok_r(NULL, " ", wordsave); *tgt = parse_range(word); } allocate(src->size * BLOCKSIZE, buffer, buffer_alloc); - size_t p = 0; - int i; - for (i = 0; i < src->count; ++i) { - check_lseek(fd, (off64_t)src->pos[i*2] * BLOCKSIZE, SEEK_SET); - size_t sz = (src->pos[i*2+1] - src->pos[i*2]) * BLOCKSIZE; - readblock(fd, *buffer+p, sz); - p += sz; + rc = ReadBlocks(src, *buffer, fd); + *src_blocks = src->size; + + free(src); + return rc; +} + +static int VerifyBlocks(const char *expected, const uint8_t *buffer, + size_t blocks, int printerror) { + char* hexdigest = NULL; + int rc = -1; + uint8_t digest[SHA_DIGEST_SIZE]; + + if (!expected || !buffer) { + return rc; } - *src_blocks = src->size; - free(src); + SHA_hash(buffer, blocks * BLOCKSIZE, digest); + hexdigest = PrintSha1(digest); + + if (hexdigest != NULL) { + rc = strcmp(expected, hexdigest); + + if (rc != 0 && printerror) { + fprintf(stderr, "failed to verify blocks (expected %s, read %s)\n", + expected, hexdigest); + } + + free(hexdigest); + } + + return rc; +} + +static char* GetStashFileName(const char* base, const char* id, const char* postfix) { + char* fn; + int len; + int res; + + if (base == NULL) { + return NULL; + } + + if (id == NULL) { + id = ""; + } + + if (postfix == NULL) { + postfix = ""; + } + + len = strlen(STASH_DIRECTORY_BASE) + 1 + strlen(base) + 1 + strlen(id) + strlen(postfix) + 1; + fn = malloc(len); + + if (fn == NULL) { + fprintf(stderr, "failed to malloc %d bytes for fn\n", len); + return NULL; + } + + res = snprintf(fn, len, STASH_DIRECTORY_BASE "/%s/%s%s", base, id, postfix); + + if (res < 0 || res >= len) { + fprintf(stderr, "failed to format file name (return value %d)\n", res); + free(fn); + return NULL; + } + + return fn; +} + +typedef void (*StashCallback)(const char*, void*); + +// Does a best effort enumeration of stash files. Ignores possible non-file +// items in the stash directory and continues despite of errors. Calls the +// 'callback' function for each file and passes 'data' to the function as a +// parameter. + +static void EnumerateStash(const char* dirname, StashCallback callback, void* data) { + char* fn; + DIR* directory; + int len; + int res; + struct dirent* item; + + if (dirname == NULL || callback == NULL) { + return; + } + + directory = opendir(dirname); + + if (directory == NULL) { + if (errno != ENOENT) { + fprintf(stderr, "failed to opendir %s (errno %d)\n", dirname, errno); + } + return; + } + + while ((item = readdir(directory)) != NULL) { + if (item->d_type != DT_REG) { + continue; + } + + len = strlen(dirname) + 1 + strlen(item->d_name) + 1; + fn = malloc(len); + + if (fn == NULL) { + fprintf(stderr, "failed to malloc %d bytes for fn\n", len); + continue; + } + + res = snprintf(fn, len, "%s/%s", dirname, item->d_name); + + if (res < 0 || res >= len) { + fprintf(stderr, "failed to format file name (return value %d)\n", res); + free(fn); + continue; + } + + callback(fn, data); + free(fn); + } + + if (closedir(directory) == -1) { + fprintf(stderr, "failed to closedir %s (errno %d)\n", dirname, errno); + } +} + +static void UpdateFileSize(const char* fn, void* data) { + int* size = (int*) data; + struct stat st; + + if (!fn || !data) { + return; + } + + if (stat(fn, &st) == -1) { + fprintf(stderr, "failed to stat %s (errno %d)\n", fn, errno); + return; + } + + *size += st.st_size; +} + +// Deletes the stash directory and all files in it. Assumes that it only +// contains files. There is nothing we can do about unlikely, but possible +// errors, so they are merely logged. + +static void DeleteFile(const char* fn, void* data) { + if (fn) { + fprintf(stderr, "deleting %s\n", fn); + + if (unlink(fn) == -1 && errno != ENOENT) { + fprintf(stderr, "failed to unlink %s (errno %d)\n", fn, errno); + } + } +} + +static void DeletePartial(const char* fn, void* data) { + if (fn && strstr(fn, ".partial") != NULL) { + DeleteFile(fn, data); + } +} + +static void DeleteStash(const char* base) { + char* dirname; + + if (base == NULL) { + return; + } + + dirname = GetStashFileName(base, NULL, NULL); + + if (dirname == NULL) { + return; + } + + fprintf(stderr, "deleting stash %s\n", base); + EnumerateStash(dirname, DeleteFile, NULL); + + if (rmdir(dirname) == -1) { + if (errno != ENOENT && errno != ENOTDIR) { + fprintf(stderr, "failed to rmdir %s (errno %d)\n", dirname, errno); + } + } + + free(dirname); +} + +static int LoadStash(const char* base, const char* id, int verify, int* blocks, uint8_t** buffer, + size_t* buffer_alloc, int printnoent) { + char *fn = NULL; + int blockcount = 0; + int fd = -1; + int rc = -1; + int res; + struct stat st; + + if (!base || !id || !buffer || !buffer_alloc) { + goto lsout; + } + + if (!blocks) { + blocks = &blockcount; + } + + fn = GetStashFileName(base, id, NULL); + + if (fn == NULL) { + goto lsout; + } + + res = stat(fn, &st); + + if (res == -1) { + if (errno != ENOENT || printnoent) { + fprintf(stderr, "failed to stat %s (errno %d)\n", fn, errno); + } + goto lsout; + } + + fprintf(stderr, " loading %s\n", fn); + + if ((st.st_size % BLOCKSIZE) != 0) { + fprintf(stderr, "%s size %zd not multiple of block size %d", fn, st.st_size, BLOCKSIZE); + goto lsout; + } + + fd = TEMP_FAILURE_RETRY(open(fn, O_RDONLY)); + + if (fd == -1) { + fprintf(stderr, "failed to open %s (errno %d)\n", fn, errno); + goto lsout; + } + + allocate(st.st_size, buffer, buffer_alloc); + + if (read_all(fd, *buffer, st.st_size) == -1) { + goto lsout; + } + + *blocks = st.st_size / BLOCKSIZE; + + if (verify && VerifyBlocks(id, *buffer, *blocks, 1) != 0) { + fprintf(stderr, "unexpected contents in %s\n", fn); + DeleteFile(fn, NULL); + goto lsout; + } + + rc = 0; + +lsout: + if (fd != -1) { + TEMP_FAILURE_RETRY(close(fd)); + } + + if (fn) { + free(fn); + } + + return rc; +} + +static int WriteStash(const char* base, const char* id, int blocks, uint8_t* buffer, + int checkspace) { + char *fn = NULL; + char *cn = NULL; + int fd = -1; + int rc = -1; + int res; + + if (base == NULL || buffer == NULL) { + goto wsout; + } + + if (checkspace && CacheSizeCheck(blocks * BLOCKSIZE) != 0) { + fprintf(stderr, "not enough space to write stash\n"); + goto wsout; + } + + fn = GetStashFileName(base, id, ".partial"); + cn = GetStashFileName(base, id, NULL); + + if (fn == NULL || cn == NULL) { + goto wsout; + } + + fprintf(stderr, " writing %d blocks to %s\n", blocks, cn); + + fd = TEMP_FAILURE_RETRY(open(fn, O_WRONLY | O_CREAT | O_TRUNC | O_SYNC, STASH_FILE_MODE)); + + if (fd == -1) { + fprintf(stderr, "failed to create %s (errno %d)\n", fn, errno); + goto wsout; + } + + if (write_all(fd, buffer, blocks * BLOCKSIZE) == -1) { + goto wsout; + } + + if (fsync(fd) == -1) { + fprintf(stderr, "failed to fsync %s (errno %d)\n", fn, errno); + goto wsout; + } + + if (rename(fn, cn) == -1) { + fprintf(stderr, "failed to rename %s to %s (errno %d)\n", fn, cn, errno); + goto wsout; + } + + rc = 0; + +wsout: + if (fd != -1) { + TEMP_FAILURE_RETRY(close(fd)); + } + + if (fn) { + free(fn); + } + + if (cn) { + free(cn); + } + + return rc; +} + +// Creates a directory for storing stash files and checks if the /cache partition +// hash enough space for the expected amount of blocks we need to store. Returns +// >0 if we created the directory, zero if it existed already, and <0 of failure. + +static int CreateStash(State* state, int maxblocks, const char* blockdev, char** base) { + char* dirname = NULL; + const uint8_t* digest; + int rc = -1; + int res; + int size = 0; + SHA_CTX ctx; + struct stat st; + + if (blockdev == NULL || base == NULL) { + goto csout; + } + + // Stash directory should be different for each partition to avoid conflicts + // when updating multiple partitions at the same time, so we use the hash of + // the block device name as the base directory + SHA_init(&ctx); + SHA_update(&ctx, blockdev, strlen(blockdev)); + digest = SHA_final(&ctx); + *base = PrintSha1(digest); + + if (*base == NULL) { + goto csout; + } + + dirname = GetStashFileName(*base, NULL, NULL); + + if (dirname == NULL) { + goto csout; + } + + res = stat(dirname, &st); + + if (res == -1 && errno != ENOENT) { + ErrorAbort(state, "failed to stat %s (errno %d)\n", dirname, errno); + goto csout; + } else if (res != 0) { + fprintf(stderr, "creating stash %s\n", dirname); + res = mkdir(dirname, STASH_DIRECTORY_MODE); + + if (res != 0) { + ErrorAbort(state, "failed to create %s (errno %d)\n", dirname, errno); + goto csout; + } + + if (CacheSizeCheck(maxblocks * BLOCKSIZE) != 0) { + ErrorAbort(state, "not enough space for stash\n"); + goto csout; + } + + rc = 1; // Created directory + goto csout; + } + + fprintf(stderr, "using existing stash %s\n", dirname); + + // If the directory already exists, calculate the space already allocated to + // stash files and check if there's enough for all required blocks. Delete any + // partially completed stash files first. + + EnumerateStash(dirname, DeletePartial, NULL); + EnumerateStash(dirname, UpdateFileSize, &size); + + size = (maxblocks * BLOCKSIZE) - size; + + if (size > 0 && CacheSizeCheck(size) != 0) { + ErrorAbort(state, "not enough space for stash (%d more needed)\n", size); + goto csout; + } + + rc = 0; // Using existing directory + +csout: + if (dirname) { + free(dirname); + } + + return rc; +} + +static int SaveStash(const char* base, char** wordsave, uint8_t** buffer, size_t* buffer_alloc, + int fd, int usehash, int* isunresumable) { + char *id = NULL; + int res = -1; + int blocks = 0; + + if (!wordsave || !buffer || !buffer_alloc || !isunresumable) { + return -1; + } + + id = strtok_r(NULL, " ", wordsave); + + if (id == NULL) { + fprintf(stderr, "missing id field in stash command\n"); + return -1; + } + + if (usehash && LoadStash(base, id, 1, &blocks, buffer, buffer_alloc, 0) == 0) { + // Stash file already exists and has expected contents. Do not + // read from source again, as the source may have been already + // overwritten during a previous attempt. + return 0; + } + + if (LoadSrcTgtVersion1(wordsave, NULL, &blocks, buffer, buffer_alloc, fd) == -1) { + return -1; + } + + if (usehash && VerifyBlocks(id, *buffer, blocks, 1) != 0) { + // Source blocks have unexpected contents. If we actually need this + // data later, this is an unrecoverable error. However, the command + // that uses the data may have already completed previously, so the + // possible failure will occur during source block verification. + fprintf(stderr, "failed to load source blocks for stash %s\n", id); + return 0; + } + + fprintf(stderr, "stashing %d blocks to %s\n", blocks, id); + return WriteStash(base, id, blocks, *buffer, 0); +} + +static int FreeStash(const char* base, const char* id) { + char *fn = NULL; + + if (base == NULL || id == NULL) { + return -1; + } + + fn = GetStashFileName(base, id, NULL); + + if (fn == NULL) { + return -1; + } + + DeleteFile(fn, NULL); + free(fn); + + return 0; } static void MoveRange(uint8_t* dest, RangeSet* locs, const uint8_t* source) { @@ -312,64 +874,613 @@ static void MoveRange(uint8_t* dest, RangeSet* locs, const uint8_t* source) { // On return, buffer is filled with the loaded source data (rearranged // and combined with stashed data as necessary). buffer may be // reallocated if needed to accommodate the source data. *tgt is the -// target RangeSet. Any stashes required are taken from stash_table -// and free()'d after being used. +// target RangeSet. Any stashes required are loaded using LoadStash. -static void LoadSrcTgtVersion2(char* wordsave, RangeSet** tgt, int* src_blocks, +static int LoadSrcTgtVersion2(char** wordsave, RangeSet** tgt, int* src_blocks, uint8_t** buffer, size_t* buffer_alloc, int fd, - uint8_t** stash_table) { + const char* stashbase, int* overlap) { char* word; + char* colonsave; + char* colon; + int id; + int res; + RangeSet* locs; + size_t stashalloc = 0; + uint8_t* stash = NULL; if (tgt != NULL) { - word = strtok_r(NULL, " ", &wordsave); + word = strtok_r(NULL, " ", wordsave); *tgt = parse_range(word); } - word = strtok_r(NULL, " ", &wordsave); + word = strtok_r(NULL, " ", wordsave); *src_blocks = strtol(word, NULL, 0); allocate(*src_blocks * BLOCKSIZE, buffer, buffer_alloc); - word = strtok_r(NULL, " ", &wordsave); + word = strtok_r(NULL, " ", wordsave); if (word[0] == '-' && word[1] == '\0') { // no source ranges, only stashes } else { RangeSet* src = parse_range(word); + res = ReadBlocks(src, *buffer, fd); - size_t p = 0; - int i; - for (i = 0; i < src->count; ++i) { - check_lseek(fd, (off64_t)src->pos[i*2] * BLOCKSIZE, SEEK_SET); - size_t sz = (src->pos[i*2+1] - src->pos[i*2]) * BLOCKSIZE; - readblock(fd, *buffer+p, sz); - p += sz; + if (overlap && tgt) { + *overlap = range_overlaps(src, *tgt); } + free(src); - word = strtok_r(NULL, " ", &wordsave); - if (word == NULL) { - // no stashes, only source range - return; + if (res == -1) { + return -1; } - RangeSet* locs = parse_range(word); + word = strtok_r(NULL, " ", wordsave); + if (word == NULL) { + // no stashes, only source range + return 0; + } + + locs = parse_range(word); MoveRange(*buffer, locs, *buffer); + free(locs); } - while ((word = strtok_r(NULL, " ", &wordsave)) != NULL) { + while ((word = strtok_r(NULL, " ", wordsave)) != NULL) { // Each word is a an index into the stash table, a colon, and // then a rangeset describing where in the source block that // stashed data should go. - char* colonsave = NULL; - char* colon = strtok_r(word, ":", &colonsave); - int stash_id = strtol(colon, NULL, 0); + colonsave = NULL; + colon = strtok_r(word, ":", &colonsave); + + res = LoadStash(stashbase, colon, 0, NULL, &stash, &stashalloc, 1); + + if (res == -1) { + // These source blocks will fail verification if used later, but we + // will let the caller decide if this is a fatal failure + fprintf(stderr, "failed to load stash %s\n", colon); + continue; + } + colon = strtok_r(NULL, ":", &colonsave); - RangeSet* locs = parse_range(colon); - MoveRange(*buffer, locs, stash_table[stash_id]); - free(stash_table[stash_id]); - stash_table[stash_id] = NULL; + locs = parse_range(colon); + + MoveRange(*buffer, locs, stash); free(locs); } + + if (stash) { + free(stash); + } + + return 0; +} + +// Parameters for transfer list command functions +typedef struct { + char* cmdname; + char* cpos; + char* freestash; + char* stashbase; + int canwrite; + int createdstash; + int fd; + int foundwrites; + int isunresumable; + int version; + int written; + NewThreadInfo nti; + pthread_t thread; + size_t bufsize; + uint8_t* buffer; + uint8_t* patch_start; +} CommandParameters; + +// Do a source/target load for move/bsdiff/imgdiff in version 3. +// +// Parameters are the same as for LoadSrcTgtVersion2, except for 'onehash', which +// tells the function whether to expect separate source and targe block hashes, or +// if they are both the same and only one hash should be expected, and +// 'isunresumable', which receives a non-zero value if block verification fails in +// a way that the update cannot be resumed anymore. +// +// If the function is unable to load the necessary blocks or their contents don't +// match the hashes, the return value is -1 and the command should be aborted. +// +// If the return value is 1, the command has already been completed according to +// the contents of the target blocks, and should not be performed again. +// +// If the return value is 0, source blocks have expected content and the command +// can be performed. + +static int LoadSrcTgtVersion3(CommandParameters* params, RangeSet** tgt, int* src_blocks, + int onehash, int* overlap) { + char* srchash = NULL; + char* tgthash = NULL; + int overlap_blocks = 0; + int rc = -1; + uint8_t* tgtbuffer = NULL; + + if (!params|| !tgt || !src_blocks || !overlap) { + goto v3out; + } + + srchash = strtok_r(NULL, " ", ¶ms->cpos); + + if (srchash == NULL) { + fprintf(stderr, "missing source hash\n"); + goto v3out; + } + + if (onehash) { + tgthash = srchash; + } else { + tgthash = strtok_r(NULL, " ", ¶ms->cpos); + + if (tgthash == NULL) { + fprintf(stderr, "missing target hash\n"); + goto v3out; + } + } + + if (LoadSrcTgtVersion2(¶ms->cpos, tgt, src_blocks, ¶ms->buffer, ¶ms->bufsize, + params->fd, params->stashbase, overlap) == -1) { + goto v3out; + } + + tgtbuffer = (uint8_t*) malloc((*tgt)->size * BLOCKSIZE); + + if (tgtbuffer == NULL) { + fprintf(stderr, "failed to allocate %d bytes\n", (*tgt)->size * BLOCKSIZE); + goto v3out; + } + + if (ReadBlocks(*tgt, tgtbuffer, params->fd) == -1) { + goto v3out; + } + + if (VerifyBlocks(tgthash, tgtbuffer, (*tgt)->size, 0) == 0) { + // Target blocks already have expected content, command should be skipped + rc = 1; + goto v3out; + } + + if (VerifyBlocks(srchash, params->buffer, *src_blocks, 1) == 0) { + // If source and target blocks overlap, stash the source blocks so we can + // resume from possible write errors + if (*overlap) { + fprintf(stderr, "stashing %d overlapping blocks to %s\n", *src_blocks, + srchash); + + if (WriteStash(params->stashbase, srchash, *src_blocks, params->buffer, 1) != 0) { + fprintf(stderr, "failed to stash overlapping source blocks\n"); + goto v3out; + } + + // Can be deleted when the write has completed + params->freestash = srchash; + } + + // Source blocks have expected content, command can proceed + rc = 0; + goto v3out; + } + + if (*overlap && LoadStash(params->stashbase, srchash, 1, NULL, ¶ms->buffer, + ¶ms->bufsize, 1) == 0) { + // Overlapping source blocks were previously stashed, command can proceed + if (params->canwrite) { + // We didn't create the stash, so delete after write only if we will + // actually perform the write + params->freestash = srchash; + } + rc = 0; + goto v3out; + } + + // Valid source data not available, update cannot be resumed + fprintf(stderr, "partition has unexpected contents\n"); + params->isunresumable = 1; + +v3out: + if (tgtbuffer) { + free(tgtbuffer); + } + + return rc; +} + +static int PerformCommandMove(CommandParameters* params) { + int blocks = 0; + int overlap = 0; + int rc = -1; + int status = 0; + RangeSet* tgt = NULL; + + if (!params) { + goto pcmout; + } + + if (params->version == 1) { + status = LoadSrcTgtVersion1(¶ms->cpos, &tgt, &blocks, ¶ms->buffer, + ¶ms->bufsize, params->fd); + } else if (params->version == 2) { + status = LoadSrcTgtVersion2(¶ms->cpos, &tgt, &blocks, ¶ms->buffer, + ¶ms->bufsize, params->fd, params->stashbase, NULL); + } else if (params->version >= 3) { + status = LoadSrcTgtVersion3(params, &tgt, &blocks, 1, &overlap); + } + + if (status == -1) { + fprintf(stderr, "failed to read blocks for move\n"); + goto pcmout; + } + + if (status == 0) { + params->foundwrites = 1; + } else if (params->foundwrites) { + fprintf(stderr, "warning: commands executed out of order [%s]\n", params->cmdname); + } + + if (params->canwrite) { + if (status == 0) { + fprintf(stderr, " moving %d blocks\n", blocks); + + if (WriteBlocks(tgt, params->buffer, params->fd) == -1) { + goto pcmout; + } + } else { + fprintf(stderr, "skipping %d already moved blocks\n", blocks); + } + + } + + if (params->freestash) { + FreeStash(params->stashbase, params->freestash); + params->freestash = NULL; + } + + params->written += tgt->size; + rc = 0; + +pcmout: + if (tgt) { + free(tgt); + } + + return rc; +} + +static int PerformCommandStash(CommandParameters* params) { + if (!params) { + return -1; + } + + return SaveStash(params->stashbase, ¶ms->cpos, ¶ms->buffer, ¶ms->bufsize, + params->fd, (params->version >= 3), ¶ms->isunresumable); +} + +static int PerformCommandFree(CommandParameters* params) { + if (!params) { + return -1; + } + + if (params->createdstash || params->canwrite) { + return FreeStash(params->stashbase, params->cpos); + } + + return 0; +} + +static int PerformCommandZero(CommandParameters* params) { + char* range = NULL; + int i; + int j; + int rc = -1; + RangeSet* tgt = NULL; + + if (!params) { + goto pczout; + } + + range = strtok_r(NULL, " ", ¶ms->cpos); + + if (range == NULL) { + fprintf(stderr, "missing target blocks for zero\n"); + goto pczout; + } + + tgt = parse_range(range); + + fprintf(stderr, " zeroing %d blocks\n", tgt->size); + + allocate(BLOCKSIZE, ¶ms->buffer, ¶ms->bufsize); + memset(params->buffer, 0, BLOCKSIZE); + + if (params->canwrite) { + for (i = 0; i < tgt->count; ++i) { + if (check_lseek(params->fd, (off64_t) tgt->pos[i * 2] * BLOCKSIZE, SEEK_SET) == -1) { + goto pczout; + } + + for (j = tgt->pos[i * 2]; j < tgt->pos[i * 2 + 1]; ++j) { + if (write_all(params->fd, params->buffer, BLOCKSIZE) == -1) { + goto pczout; + } + } + } + } + + if (params->cmdname[0] == 'z') { + // Update only for the zero command, as the erase command will call + // this if DEBUG_ERASE is defined. + params->written += tgt->size; + } + + rc = 0; + +pczout: + if (tgt) { + free(tgt); + } + + return rc; +} + +static int PerformCommandNew(CommandParameters* params) { + char* range = NULL; + int rc = -1; + RangeSet* tgt = NULL; + RangeSinkState rss; + + if (!params) { + goto pcnout; + } + + range = strtok_r(NULL, " ", ¶ms->cpos); + + if (range == NULL) { + goto pcnout; + } + + tgt = parse_range(range); + + if (params->canwrite) { + fprintf(stderr, " writing %d blocks of new data\n", tgt->size); + + rss.fd = params->fd; + rss.tgt = tgt; + rss.p_block = 0; + rss.p_remain = (tgt->pos[1] - tgt->pos[0]) * BLOCKSIZE; + + if (check_lseek(params->fd, (off64_t) tgt->pos[0] * BLOCKSIZE, SEEK_SET) == -1) { + goto pcnout; + } + + pthread_mutex_lock(¶ms->nti.mu); + params->nti.rss = &rss; + pthread_cond_broadcast(¶ms->nti.cv); + + while (params->nti.rss) { + pthread_cond_wait(¶ms->nti.cv, ¶ms->nti.mu); + } + + pthread_mutex_unlock(¶ms->nti.mu); + } + + params->written += tgt->size; + rc = 0; + +pcnout: + if (tgt) { + free(tgt); + } + + return rc; +} + +static int PerformCommandDiff(CommandParameters* params) { + char* logparams = NULL; + char* value = NULL; + int blocks = 0; + int overlap = 0; + int rc = -1; + int status = 0; + RangeSet* tgt = NULL; + RangeSinkState rss; + size_t len = 0; + size_t offset = 0; + Value patch_value; + + if (!params) { + goto pcdout; + } + + logparams = strdup(params->cpos); + value = strtok_r(NULL, " ", ¶ms->cpos); + + if (value == NULL) { + fprintf(stderr, "missing patch offset for %s\n", params->cmdname); + goto pcdout; + } + + offset = strtoul(value, NULL, 0); + + value = strtok_r(NULL, " ", ¶ms->cpos); + + if (value == NULL) { + fprintf(stderr, "missing patch length for %s\n", params->cmdname); + goto pcdout; + } + + len = strtoul(value, NULL, 0); + + if (params->version == 1) { + status = LoadSrcTgtVersion1(¶ms->cpos, &tgt, &blocks, ¶ms->buffer, + ¶ms->bufsize, params->fd); + } else if (params->version == 2) { + status = LoadSrcTgtVersion2(¶ms->cpos, &tgt, &blocks, ¶ms->buffer, + ¶ms->bufsize, params->fd, params->stashbase, NULL); + } else if (params->version >= 3) { + status = LoadSrcTgtVersion3(params, &tgt, &blocks, 0, &overlap); + } + + if (status == -1) { + fprintf(stderr, "failed to read blocks for diff\n"); + goto pcdout; + } + + if (status == 0) { + params->foundwrites = 1; + } else if (params->foundwrites) { + fprintf(stderr, "warning: commands executed out of order [%s]\n", params->cmdname); + } + + if (params->canwrite) { + if (status == 0) { + fprintf(stderr, "patching %d blocks to %d\n", blocks, tgt->size); + + patch_value.type = VAL_BLOB; + patch_value.size = len; + patch_value.data = (char*) (params->patch_start + offset); + + rss.fd = params->fd; + rss.tgt = tgt; + rss.p_block = 0; + rss.p_remain = (tgt->pos[1] - tgt->pos[0]) * BLOCKSIZE; + + if (check_lseek(params->fd, (off64_t) tgt->pos[0] * BLOCKSIZE, SEEK_SET) == -1) { + goto pcdout; + } + + if (params->cmdname[0] == 'i') { // imgdiff + ApplyImagePatch(params->buffer, blocks * BLOCKSIZE, &patch_value, + &RangeSinkWrite, &rss, NULL, NULL); + } else { + ApplyBSDiffPatch(params->buffer, blocks * BLOCKSIZE, &patch_value, + 0, &RangeSinkWrite, &rss, NULL); + } + + // We expect the output of the patcher to fill the tgt ranges exactly. + if (rss.p_block != tgt->count || rss.p_remain != 0) { + fprintf(stderr, "range sink underrun?\n"); + } + } else { + fprintf(stderr, "skipping %d blocks already patched to %d [%s]\n", + blocks, tgt->size, logparams); + } + } + + if (params->freestash) { + FreeStash(params->stashbase, params->freestash); + params->freestash = NULL; + } + + params->written += tgt->size; + rc = 0; + +pcdout: + if (logparams) { + free(logparams); + } + + if (tgt) { + free(tgt); + } + + return rc; +} + +static int PerformCommandErase(CommandParameters* params) { + char* range = NULL; + int i; + int rc = -1; + RangeSet* tgt = NULL; + struct stat st; + uint64_t blocks[2]; + + if (DEBUG_ERASE) { + return PerformCommandZero(params); + } + + if (!params) { + goto pceout; + } + + if (fstat(params->fd, &st) == -1) { + fprintf(stderr, "failed to fstat device to erase (errno %d)\n", errno); + goto pceout; + } + + if (!S_ISBLK(st.st_mode)) { + fprintf(stderr, "not a block device; skipping erase\n"); + rc = 0; + goto pceout; + } + + range = strtok_r(NULL, " ", ¶ms->cpos); + + if (range == NULL) { + fprintf(stderr, "missing target blocks for zero\n"); + goto pceout; + } + + tgt = parse_range(range); + + if (params->canwrite) { + fprintf(stderr, " erasing %d blocks\n", tgt->size); + + for (i = 0; i < tgt->count; ++i) { + // offset in bytes + blocks[0] = tgt->pos[i * 2] * (uint64_t) BLOCKSIZE; + // length in bytes + blocks[1] = (tgt->pos[i * 2 + 1] - tgt->pos[i * 2]) * (uint64_t) BLOCKSIZE; + + if (ioctl(params->fd, BLKDISCARD, &blocks) == -1) { + fprintf(stderr, "failed to blkdiscard (errno %d)\n", errno); + // Continue anyway, nothing we can do + } + } + } + + rc = 0; + +pceout: + if (tgt) { + free(tgt); + } + + return rc; +} + +// Definitions for transfer list command functions +typedef int (*CommandFunction)(CommandParameters*); + +typedef struct { + const char* name; + CommandFunction f; +} Command; + +// CompareCommands and CompareCommandNames are for the hash table + +static int CompareCommands(const void* c1, const void* c2) { + return strcmp(((const Command*) c1)->name, ((const Command*) c2)->name); +} + +static int CompareCommandNames(const void* c1, const void* c2) { + return strcmp(((const Command*) c1)->name, (const char*) c2); +} + +// HashString is used to hash command names for the hash table + +static unsigned int HashString(const char *s) { + unsigned int hash = 0; + if (s) { + while (*s) { + hash = hash * 33 + *s++; + } + } + return hash; } // args: @@ -378,379 +1489,371 @@ static void LoadSrcTgtVersion2(char* wordsave, RangeSet** tgt, int* src_blocks, // - new data stream (filename within package.zip) // - patch stream (filename within package.zip, must be uncompressed) -Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[]) { - Value* blockdev_filename; - Value* transfer_list_value; +static Value* PerformBlockImageUpdate(const char* name, State* state, int argc, Expr* argv[], + const Command* commands, int cmdcount, int dryrun) { + + char* line = NULL; + char* linesave = NULL; + char* logcmd = NULL; char* transfer_list = NULL; - Value* new_data_fn; - Value* patch_data_fn; - bool success = false; + CommandParameters params; + const Command* cmd = NULL; + const ZipEntry* new_entry = NULL; + const ZipEntry* patch_entry = NULL; + FILE* cmd_pipe = NULL; + HashTable* cmdht = NULL; + int i; + int res; + int rc = -1; + int stash_max_blocks = 0; + int total_blocks = 0; + pthread_attr_t attr; + unsigned int cmdhash; + UpdaterInfo* ui = NULL; + Value* blockdev_filename = NULL; + Value* new_data_fn = NULL; + Value* patch_data_fn = NULL; + Value* transfer_list_value = NULL; + ZipArchive* za = NULL; + + memset(¶ms, 0, sizeof(params)); + params.canwrite = !dryrun; + + fprintf(stderr, "performing %s\n", dryrun ? "verification" : "update"); if (ReadValueArgs(state, argv, 4, &blockdev_filename, &transfer_list_value, - &new_data_fn, &patch_data_fn) < 0) { - return NULL; + &new_data_fn, &patch_data_fn) < 0) { + goto pbiudone; } if (blockdev_filename->type != VAL_STRING) { ErrorAbort(state, "blockdev_filename argument to %s must be string", name); - goto done; + goto pbiudone; } if (transfer_list_value->type != VAL_BLOB) { ErrorAbort(state, "transfer_list argument to %s must be blob", name); - goto done; + goto pbiudone; } if (new_data_fn->type != VAL_STRING) { ErrorAbort(state, "new_data_fn argument to %s must be string", name); - goto done; + goto pbiudone; } if (patch_data_fn->type != VAL_STRING) { ErrorAbort(state, "patch_data_fn argument to %s must be string", name); - goto done; + goto pbiudone; } - UpdaterInfo* ui = (UpdaterInfo*)(state->cookie); - FILE* cmd_pipe = ui->cmd_pipe; + ui = (UpdaterInfo*) state->cookie; - ZipArchive* za = ((UpdaterInfo*)(state->cookie))->package_zip; + if (ui == NULL) { + goto pbiudone; + } + + cmd_pipe = ui->cmd_pipe; + za = ui->package_zip; + + if (cmd_pipe == NULL || za == NULL) { + goto pbiudone; + } + + patch_entry = mzFindZipEntry(za, patch_data_fn->data); - const ZipEntry* patch_entry = mzFindZipEntry(za, patch_data_fn->data); if (patch_entry == NULL) { - ErrorAbort(state, "%s(): no file \"%s\" in package", name, patch_data_fn->data); - goto done; + fprintf(stderr, "%s(): no file \"%s\" in package", name, patch_data_fn->data); + goto pbiudone; } - uint8_t* patch_start = ((UpdaterInfo*)(state->cookie))->package_zip_addr + - mzGetZipEntryOffset(patch_entry); + params.patch_start = ui->package_zip_addr + mzGetZipEntryOffset(patch_entry); + new_entry = mzFindZipEntry(za, new_data_fn->data); - const ZipEntry* new_entry = mzFindZipEntry(za, new_data_fn->data); if (new_entry == NULL) { - ErrorAbort(state, "%s(): no file \"%s\" in package", name, new_data_fn->data); - goto done; + fprintf(stderr, "%s(): no file \"%s\" in package", name, new_data_fn->data); + goto pbiudone; } - // The transfer list is a text file containing commands to - // transfer data from one place to another on the target - // partition. We parse it and execute the commands in order: - // - // zero [rangeset] - // - fill the indicated blocks with zeros - // - // new [rangeset] - // - fill the blocks with data read from the new_data file - // - // erase [rangeset] - // - mark the given blocks as empty - // - // move <...> - // bsdiff <...> - // imgdiff <...> - // - read the source blocks, apply a patch (or not in the - // case of move), write result to target blocks. bsdiff or - // imgdiff specifies the type of patch; move means no patch - // at all. - // - // The format of <...> differs between versions 1 and 2; - // see the LoadSrcTgtVersion{1,2}() functions for a - // description of what's expected. - // - // stash - // - (version 2 only) load the given source range and stash - // the data in the given slot of the stash table. - // - // The creator of the transfer list will guarantee that no block - // is read (ie, used as the source for a patch or move) after it - // has been written. - // - // In version 2, the creator will guarantee that a given stash is - // loaded (with a stash command) before it's used in a - // move/bsdiff/imgdiff command. - // - // Within one command the source and target ranges may overlap so - // in general we need to read the entire source into memory before - // writing anything to the target blocks. - // - // All the patch data is concatenated into one patch_data file in - // the update package. It must be stored uncompressed because we - // memory-map it in directly from the archive. (Since patches are - // already compressed, we lose very little by not compressing - // their concatenation.) + params.fd = TEMP_FAILURE_RETRY(open(blockdev_filename->data, O_RDWR)); - pthread_t new_data_thread; - NewThreadInfo nti; - nti.za = za; - nti.entry = new_entry; - nti.rss = NULL; - pthread_mutex_init(&nti.mu, NULL); - pthread_cond_init(&nti.cv, NULL); - - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - pthread_create(&new_data_thread, &attr, unzip_new_data, &nti); - - int i, j; - - char* linesave; - char* wordsave; - - int fd = open(blockdev_filename->data, O_RDWR); - if (fd < 0) { - ErrorAbort(state, "failed to open %s: %s", blockdev_filename->data, strerror(errno)); - goto done; + if (params.fd == -1) { + fprintf(stderr, "failed to open %s: %s", blockdev_filename->data, strerror(errno)); + goto pbiudone; } - char* line; - char* word; + if (params.canwrite) { + params.nti.za = za; + params.nti.entry = new_entry; + + pthread_mutex_init(¶ms.nti.mu, NULL); + pthread_cond_init(¶ms.nti.cv, NULL); + pthread_attr_init(&attr); + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); + + if (pthread_create(¶ms.thread, &attr, unzip_new_data, ¶ms.nti) != 0) { + fprintf(stderr, "failed to create a thread (errno %d)\n", errno); + goto pbiudone; + } + } + + // The data in transfer_list_value is not necessarily null-terminated, so we need + // to copy it to a new buffer and add the null that strtok_r will need. + transfer_list = malloc(transfer_list_value->size + 1); - // The data in transfer_list_value is not necessarily - // null-terminated, so we need to copy it to a new buffer and add - // the null that strtok_r will need. - transfer_list = malloc(transfer_list_value->size+1); if (transfer_list == NULL) { fprintf(stderr, "failed to allocate %zd bytes for transfer list\n", - transfer_list_value->size+1); - exit(1); + transfer_list_value->size + 1); + goto pbiudone; } + memcpy(transfer_list, transfer_list_value->data, transfer_list_value->size); transfer_list[transfer_list_value->size] = '\0'; + // First line in transfer list is the version number line = strtok_r(transfer_list, "\n", &linesave); + params.version = strtol(line, NULL, 0); - int version; - // first line in transfer list is the version number; currently - // there's only version 1. - if (strcmp(line, "1") == 0) { - version = 1; - } else if (strcmp(line, "2") == 0) { - version = 2; - } else { - ErrorAbort(state, "unexpected transfer list version [%s]\n", line); - goto done; + if (params.version < 1 || params.version > 3) { + fprintf(stderr, "unexpected transfer list version [%s]\n", line); + goto pbiudone; } - printf("blockimg version is %d\n", version); - // second line in transfer list is the total number of blocks we - // expect to write. + fprintf(stderr, "blockimg version is %d\n", params.version); + + // Second line in transfer list is the total number of blocks we expect to write line = strtok_r(NULL, "\n", &linesave); - int total_blocks = strtol(line, NULL, 0); - // shouldn't happen, but avoid divide by zero. - if (total_blocks == 0) ++total_blocks; - int blocks_so_far = 0; + total_blocks = strtol(line, NULL, 0); - uint8_t** stash_table = NULL; - if (version >= 2) { - // Next line is how many stash entries are needed simultaneously. + if (total_blocks < 0) { + ErrorAbort(state, "unexpected block count [%s]\n", line); + goto pbiudone; + } else if (total_blocks == 0) { + rc = 0; + goto pbiudone; + } + + if (params.version >= 2) { + // Third line is how many stash entries are needed simultaneously line = strtok_r(NULL, "\n", &linesave); - int stash_entries = strtol(line, NULL, 0); + fprintf(stderr, "maximum stash entries %s\n", line); - stash_table = (uint8_t**) calloc(stash_entries, sizeof(uint8_t*)); - if (stash_table == NULL) { - fprintf(stderr, "failed to allocate %d-entry stash table\n", stash_entries); - exit(1); + // Fourth line is the maximum number of blocks that will be stashed simultaneously + line = strtok_r(NULL, "\n", &linesave); + stash_max_blocks = strtol(line, NULL, 0); + + if (stash_max_blocks < 0) { + ErrorAbort(state, "unexpected maximum stash blocks [%s]\n", line); + goto pbiudone; } - // Next line is the maximum number of blocks that will be - // stashed simultaneously. This could be used to verify that - // enough memory or scratch disk space is available. - line = strtok_r(NULL, "\n", &linesave); - int stash_max_blocks = strtol(line, NULL, 0); + if (stash_max_blocks > 0) { + res = CreateStash(state, stash_max_blocks, blockdev_filename->data, + ¶ms.stashbase); + + if (res == -1) { + goto pbiudone; + } + + params.createdstash = res; + } } - uint8_t* buffer = NULL; - size_t buffer_alloc = 0; + // Build a hash table of the available commands + cmdht = mzHashTableCreate(cmdcount, NULL); - // third and subsequent lines are all individual transfer commands. + for (i = 0; i < cmdcount; ++i) { + cmdhash = HashString(commands[i].name); + mzHashTableLookup(cmdht, cmdhash, (void*) &commands[i], CompareCommands, true); + } + + // Subsequent lines are all individual transfer commands for (line = strtok_r(NULL, "\n", &linesave); line; line = strtok_r(NULL, "\n", &linesave)) { - char* style; - style = strtok_r(line, " ", &wordsave); + logcmd = strdup(line); + params.cmdname = strtok_r(line, " ", ¶ms.cpos); - if (strcmp("move", style) == 0) { - RangeSet* tgt; - int src_blocks; - if (version == 1) { - LoadSrcTgtVersion1(wordsave, &tgt, &src_blocks, - &buffer, &buffer_alloc, fd); - } else if (version == 2) { - LoadSrcTgtVersion2(wordsave, &tgt, &src_blocks, - &buffer, &buffer_alloc, fd, stash_table); - } + if (params.cmdname == NULL) { + fprintf(stderr, "missing command [%s]\n", line); + goto pbiudone; + } - printf(" moving %d blocks\n", src_blocks); + cmdhash = HashString(params.cmdname); + cmd = (const Command*) mzHashTableLookup(cmdht, cmdhash, params.cmdname, + CompareCommandNames, false); - size_t p = 0; - for (i = 0; i < tgt->count; ++i) { - check_lseek(fd, (off64_t)tgt->pos[i*2] * BLOCKSIZE, SEEK_SET); - size_t sz = (tgt->pos[i*2+1] - tgt->pos[i*2]) * BLOCKSIZE; - writeblock(fd, buffer+p, sz); - p += sz; - } + if (cmd == NULL) { + fprintf(stderr, "unexpected command [%s]\n", params.cmdname); + goto pbiudone; + } - blocks_so_far += tgt->size; - fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); + if (cmd->f != NULL && cmd->f(¶ms) == -1) { + fprintf(stderr, "failed to execute command [%s]\n", + logcmd ? logcmd : params.cmdname); + goto pbiudone; + } + + if (logcmd) { + free(logcmd); + logcmd = NULL; + } + + if (params.canwrite) { + fprintf(cmd_pipe, "set_progress %.4f\n", (double) params.written / total_blocks); fflush(cmd_pipe); - - free(tgt); - - } else if (strcmp("stash", style) == 0) { - word = strtok_r(NULL, " ", &wordsave); - int stash_id = strtol(word, NULL, 0); - int src_blocks; - size_t stash_alloc = 0; - - // Even though the "stash" style only appears in version - // 2, the version 1 source loader happens to do exactly - // what we want to read data into the stash_table. - LoadSrcTgtVersion1(wordsave, NULL, &src_blocks, - stash_table + stash_id, &stash_alloc, fd); - - } else if (strcmp("zero", style) == 0 || - (DEBUG_ERASE && strcmp("erase", style) == 0)) { - word = strtok_r(NULL, " ", &wordsave); - RangeSet* tgt = parse_range(word); - - printf(" zeroing %d blocks\n", tgt->size); - - allocate(BLOCKSIZE, &buffer, &buffer_alloc); - memset(buffer, 0, BLOCKSIZE); - for (i = 0; i < tgt->count; ++i) { - check_lseek(fd, (off64_t)tgt->pos[i*2] * BLOCKSIZE, SEEK_SET); - for (j = tgt->pos[i*2]; j < tgt->pos[i*2+1]; ++j) { - writeblock(fd, buffer, BLOCKSIZE); - } - } - - if (style[0] == 'z') { // "zero" but not "erase" - blocks_so_far += tgt->size; - fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); - fflush(cmd_pipe); - } - - free(tgt); - } else if (strcmp("new", style) == 0) { - - word = strtok_r(NULL, " ", &wordsave); - RangeSet* tgt = parse_range(word); - - printf(" writing %d blocks of new data\n", tgt->size); - - RangeSinkState rss; - rss.fd = fd; - rss.tgt = tgt; - rss.p_block = 0; - rss.p_remain = (tgt->pos[1] - tgt->pos[0]) * BLOCKSIZE; - check_lseek(fd, (off64_t)tgt->pos[0] * BLOCKSIZE, SEEK_SET); - - pthread_mutex_lock(&nti.mu); - nti.rss = &rss; - pthread_cond_broadcast(&nti.cv); - while (nti.rss) { - pthread_cond_wait(&nti.cv, &nti.mu); - } - pthread_mutex_unlock(&nti.mu); - - blocks_so_far += tgt->size; - fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); - fflush(cmd_pipe); - - free(tgt); - - } else if (strcmp("bsdiff", style) == 0 || - strcmp("imgdiff", style) == 0) { - word = strtok_r(NULL, " ", &wordsave); - size_t patch_offset = strtoul(word, NULL, 0); - word = strtok_r(NULL, " ", &wordsave); - size_t patch_len = strtoul(word, NULL, 0); - - RangeSet* tgt; - int src_blocks; - if (version == 1) { - LoadSrcTgtVersion1(wordsave, &tgt, &src_blocks, - &buffer, &buffer_alloc, fd); - } else if (version == 2) { - LoadSrcTgtVersion2(wordsave, &tgt, &src_blocks, - &buffer, &buffer_alloc, fd, stash_table); - } - - printf(" patching %d blocks to %d\n", src_blocks, tgt->size); - - Value patch_value; - patch_value.type = VAL_BLOB; - patch_value.size = patch_len; - patch_value.data = (char*)(patch_start + patch_offset); - - RangeSinkState rss; - rss.fd = fd; - rss.tgt = tgt; - rss.p_block = 0; - rss.p_remain = (tgt->pos[1] - tgt->pos[0]) * BLOCKSIZE; - check_lseek(fd, (off64_t)tgt->pos[0] * BLOCKSIZE, SEEK_SET); - - if (style[0] == 'i') { // imgdiff - ApplyImagePatch(buffer, src_blocks * BLOCKSIZE, - &patch_value, - &RangeSinkWrite, &rss, NULL, NULL); - } else { - ApplyBSDiffPatch(buffer, src_blocks * BLOCKSIZE, - &patch_value, 0, - &RangeSinkWrite, &rss, NULL); - } - - // We expect the output of the patcher to fill the tgt ranges exactly. - if (rss.p_block != tgt->count || rss.p_remain != 0) { - fprintf(stderr, "range sink underrun?\n"); - } - - blocks_so_far += tgt->size; - fprintf(cmd_pipe, "set_progress %.4f\n", (double)blocks_so_far / total_blocks); - fflush(cmd_pipe); - - free(tgt); - } else if (!DEBUG_ERASE && strcmp("erase", style) == 0) { - struct stat st; - if (fstat(fd, &st) == 0 && S_ISBLK(st.st_mode)) { - word = strtok_r(NULL, " ", &wordsave); - RangeSet* tgt = parse_range(word); - - printf(" erasing %d blocks\n", tgt->size); - - for (i = 0; i < tgt->count; ++i) { - uint64_t range[2]; - // offset in bytes - range[0] = tgt->pos[i*2] * (uint64_t)BLOCKSIZE; - // len in bytes - range[1] = (tgt->pos[i*2+1] - tgt->pos[i*2]) * (uint64_t)BLOCKSIZE; - - if (ioctl(fd, BLKDISCARD, &range) < 0) { - printf(" blkdiscard failed: %s\n", strerror(errno)); - } - } - - free(tgt); - } else { - printf(" ignoring erase (not block device)\n"); - } - } else { - fprintf(stderr, "unknown transfer style \"%s\"\n", style); - exit(1); } } - pthread_join(new_data_thread, NULL); - success = true; + if (params.canwrite) { + pthread_join(params.thread, NULL); - free(buffer); - printf("wrote %d blocks; expected %d\n", blocks_so_far, total_blocks); - printf("max alloc needed was %zu\n", buffer_alloc); + fprintf(stderr, "wrote %d blocks; expected %d\n", params.written, total_blocks); + fprintf(stderr, "max alloc needed was %zu\n", params.bufsize); - done: - free(transfer_list); - FreeValue(blockdev_filename); - FreeValue(transfer_list_value); - FreeValue(new_data_fn); - FreeValue(patch_data_fn); - return StringValue(success ? strdup("t") : strdup("")); + // Delete stash only after successfully completing the update, as it + // may contain blocks needed to complete the update later. + DeleteStash(params.stashbase); + } else { + fprintf(stderr, "verified partition contents; update may be resumed\n"); + } + + rc = 0; + +pbiudone: + if (params.fd != -1) { + if (fsync(params.fd) == -1) { + fprintf(stderr, "failed to fsync device (errno %d)\n", errno); + } + TEMP_FAILURE_RETRY(close(params.fd)); + } + + if (logcmd) { + free(logcmd); + } + + if (cmdht) { + mzHashTableFree(cmdht); + } + + if (params.buffer) { + free(params.buffer); + } + + if (transfer_list) { + free(transfer_list); + } + + if (blockdev_filename) { + FreeValue(blockdev_filename); + } + + if (transfer_list_value) { + FreeValue(transfer_list_value); + } + + if (new_data_fn) { + FreeValue(new_data_fn); + } + + if (patch_data_fn) { + FreeValue(patch_data_fn); + } + + // Only delete the stash if the update cannot be resumed, or it's + // a verification run and we created the stash. + if (params.isunresumable || (!params.canwrite && params.createdstash)) { + DeleteStash(params.stashbase); + } + + if (params.stashbase) { + free(params.stashbase); + } + + return StringValue(rc == 0 ? strdup("t") : strdup("")); +} + +// The transfer list is a text file containing commands to +// transfer data from one place to another on the target +// partition. We parse it and execute the commands in order: +// +// zero [rangeset] +// - fill the indicated blocks with zeros +// +// new [rangeset] +// - fill the blocks with data read from the new_data file +// +// erase [rangeset] +// - mark the given blocks as empty +// +// move <...> +// bsdiff <...> +// imgdiff <...> +// - read the source blocks, apply a patch (or not in the +// case of move), write result to target blocks. bsdiff or +// imgdiff specifies the type of patch; move means no patch +// at all. +// +// The format of <...> differs between versions 1 and 2; +// see the LoadSrcTgtVersion{1,2}() functions for a +// description of what's expected. +// +// stash +// - (version 2+ only) load the given source range and stash +// the data in the given slot of the stash table. +// +// The creator of the transfer list will guarantee that no block +// is read (ie, used as the source for a patch or move) after it +// has been written. +// +// In version 2, the creator will guarantee that a given stash is +// loaded (with a stash command) before it's used in a +// move/bsdiff/imgdiff command. +// +// Within one command the source and target ranges may overlap so +// in general we need to read the entire source into memory before +// writing anything to the target blocks. +// +// All the patch data is concatenated into one patch_data file in +// the update package. It must be stored uncompressed because we +// memory-map it in directly from the archive. (Since patches are +// already compressed, we lose very little by not compressing +// their concatenation.) +// +// In version 3, commands that read data from the partition (i.e. +// move/bsdiff/imgdiff/stash) have one or more additional hashes +// before the range parameters, which are used to check if the +// command has already been completed and verify the integrity of +// the source data. + +Value* BlockImageVerifyFn(const char* name, State* state, int argc, Expr* argv[]) { + // Commands which are not tested are set to NULL to skip them completely + const Command commands[] = { + { "bsdiff", PerformCommandDiff }, + { "erase", NULL }, + { "free", PerformCommandFree }, + { "imgdiff", PerformCommandDiff }, + { "move", PerformCommandMove }, + { "new", NULL }, + { "stash", PerformCommandStash }, + { "zero", NULL } + }; + + // Perform a dry run without writing to test if an update can proceed + return PerformBlockImageUpdate(name, state, argc, argv, commands, + sizeof(commands) / sizeof(commands[0]), 1); +} + +Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[]) { + const Command commands[] = { + { "bsdiff", PerformCommandDiff }, + { "erase", PerformCommandErase }, + { "free", PerformCommandFree }, + { "imgdiff", PerformCommandDiff }, + { "move", PerformCommandMove }, + { "new", PerformCommandNew }, + { "stash", PerformCommandStash }, + { "zero", PerformCommandZero } + }; + + return PerformBlockImageUpdate(name, state, argc, argv, commands, + sizeof(commands) / sizeof(commands[0]), 0); } Value* RangeSha1Fn(const char* name, State* state, int argc, Expr* argv[]) { @@ -784,9 +1887,19 @@ Value* RangeSha1Fn(const char* name, State* state, int argc, Expr* argv[]) { int i, j; for (i = 0; i < rs->count; ++i) { - check_lseek(fd, (off64_t)rs->pos[i*2] * BLOCKSIZE, SEEK_SET); + if (check_lseek(fd, (off64_t)rs->pos[i*2] * BLOCKSIZE, SEEK_SET) == -1) { + ErrorAbort(state, "failed to seek %s: %s", blockdev_filename->data, + strerror(errno)); + goto done; + } + for (j = rs->pos[i*2]; j < rs->pos[i*2+1]; ++j) { - readblock(fd, buffer, BLOCKSIZE); + if (read_all(fd, buffer, BLOCKSIZE) == -1) { + ErrorAbort(state, "failed to read %s: %s", blockdev_filename->data, + strerror(errno)); + goto done; + } + SHA_update(&ctx, buffer, BLOCKSIZE); } } @@ -804,6 +1917,7 @@ Value* RangeSha1Fn(const char* name, State* state, int argc, Expr* argv[]) { } void RegisterBlockImageFunctions() { + RegisterFunction("block_image_verify", BlockImageVerifyFn); RegisterFunction("block_image_update", BlockImageUpdateFn); RegisterFunction("range_sha1", RangeSha1Fn); } From 1df64d3278378d0d234be11cfeafec5e6a96bd2d Mon Sep 17 00:00:00 2001 From: Jesse Zhao Date: Tue, 17 Feb 2015 17:09:23 -0800 Subject: [PATCH 09/13] Initialize stashbase even stash_max_blocks = 0 Change-Id: I480c02ffedd811f4dda9940ef979a05ff54f1435 Bug: 19410117 --- updater/blockimg.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/updater/blockimg.c b/updater/blockimg.c index a0f81e93..9c39634f 100644 --- a/updater/blockimg.c +++ b/updater/blockimg.c @@ -1643,7 +1643,7 @@ static Value* PerformBlockImageUpdate(const char* name, State* state, int argc, goto pbiudone; } - if (stash_max_blocks > 0) { + if (stash_max_blocks >= 0) { res = CreateStash(state, stash_max_blocks, blockdev_filename->data, ¶ms.stashbase); From 3e91f691a6bf6b154ea98982b073e84e86328baf Mon Sep 17 00:00:00 2001 From: Jesse Zhao Date: Thu, 8 Jan 2015 15:59:23 -0800 Subject: [PATCH 10/13] Bump up max_map_count value. Change-Id: Id3e2c0795b817db9a85bc84cba2aa05d20179d39 Bug: 18503789 (cherry picked from commit 5bf74b238b402eaaf8c5bd1663fe4d592e59421f) --- etc/init.rc | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/init.rc b/etc/init.rc index 1b402e20..8ff72809 100644 --- a/etc/init.rc +++ b/etc/init.rc @@ -30,6 +30,7 @@ on init chmod 0775 /tmp write /proc/sys/kernel/panic_on_oops 1 + write /proc/sys/vm/max_map_count 1000000 on fs mkdir /dev/usb-ffs 0770 shell shell From 9c0f5d6b348e37533bdcccf1166d6cbf1ca5c50b Mon Sep 17 00:00:00 2001 From: Narayan Kamath Date: Mon, 23 Feb 2015 14:09:31 +0000 Subject: [PATCH 11/13] Remove more dead code from minzip. I've added explanatory comments to mzExtractRecursive because that function will live on as a utility even after we move the zip format related logic to libziparchive. bug: 19472796 (cherry-picked from commit c9ccdfd7a42de08c47ab771b94dc5b9d1f957b95) Change-Id: I8b7fb6fa3eafb2e7ac080ef7a7eceb691b252d8a --- minzip/Zip.c | 178 +++++++++++++++++----------------------------- minzip/Zip.h | 13 ++-- updater/install.c | 2 +- 3 files changed, 72 insertions(+), 121 deletions(-) diff --git a/minzip/Zip.c b/minzip/Zip.c index 12e06f6d..d3ff79be 100644 --- a/minzip/Zip.c +++ b/minzip/Zip.c @@ -828,7 +828,7 @@ static const char *targetEntryPath(MzPathHelper *helper, ZipEntry *pEntry) */ bool mzExtractRecursive(const ZipArchive *pArchive, const char *zipDir, const char *targetDir, - int flags, const struct utimbuf *timestamp, + const struct utimbuf *timestamp, void (*callback)(const char *fn, void *), void *cookie, struct selabel_handle *sehnd) { @@ -932,30 +932,19 @@ bool mzExtractRecursive(const ZipArchive *pArchive, break; } - /* With DRY_RUN set, invoke the callback but don't do anything else. - */ - if (flags & MZ_EXTRACT_DRY_RUN) { - if (callback != NULL) callback(targetFile, cookie); - continue; - } - - /* Create the file or directory. - */ #define UNZIP_DIRMODE 0755 #define UNZIP_FILEMODE 0644 - if (pEntry->fileName[pEntry->fileNameLen-1] == '/') { - if (!(flags & MZ_EXTRACT_FILES_ONLY)) { - int ret = dirCreateHierarchy( - targetFile, UNZIP_DIRMODE, timestamp, false, sehnd); - if (ret != 0) { - LOGE("Can't create containing directory for \"%s\": %s\n", - targetFile, strerror(errno)); - ok = false; - break; - } - LOGD("Extracted dir \"%s\"\n", targetFile); - } - } else { + /* + * Create the file or directory. We ignore directory entries + * because we recursively create paths to each file entry we encounter + * in the zip archive anyway. + * + * NOTE: A "directory entry" in a zip archive is just a zero length + * entry that ends in a "/". They're not mandatory and many tools get + * rid of them. We need to process them only if we want to preserve + * empty directories from the archive. + */ + if (pEntry->fileName[pEntry->fileNameLen-1] != '/') { /* This is not a directory. First, make sure that * the containing directory exists. */ @@ -968,97 +957,62 @@ bool mzExtractRecursive(const ZipArchive *pArchive, break; } - /* With FILES_ONLY set, we need to ignore metadata entirely, - * so treat symlinks as regular files. + /* + * The entry is a regular file or a symlink. Open the target for writing. + * + * TODO: This behavior for symlinks seems rather bizarre. For a + * symlink foo/bar/baz -> foo/tar/taz, we will create a file called + * "foo/bar/baz" whose contents are the literal "foo/tar/taz". We + * warn about this for now and preserve older behavior. */ - if (!(flags & MZ_EXTRACT_FILES_ONLY) && mzIsZipEntrySymlink(pEntry)) { - /* The entry is a symbolic link. - * The relative target of the symlink is in the - * data section of this entry. - */ - if (pEntry->uncompLen == 0) { - LOGE("Symlink entry \"%s\" has no target\n", - targetFile); - ok = false; - break; - } - char *linkTarget = malloc(pEntry->uncompLen + 1); - if (linkTarget == NULL) { - ok = false; - break; - } - ok = mzReadZipEntry(pArchive, pEntry, linkTarget, - pEntry->uncompLen); - if (!ok) { - LOGE("Can't read symlink target for \"%s\"\n", - targetFile); - free(linkTarget); - break; - } - linkTarget[pEntry->uncompLen] = '\0'; - - /* Make the link. - */ - ret = symlink(linkTarget, targetFile); - if (ret != 0) { - LOGE("Can't symlink \"%s\" to \"%s\": %s\n", - targetFile, linkTarget, strerror(errno)); - free(linkTarget); - ok = false; - break; - } - LOGD("Extracted symlink \"%s\" -> \"%s\"\n", - targetFile, linkTarget); - free(linkTarget); - } else { - /* The entry is a regular file. - * Open the target for writing. - */ - - char *secontext = NULL; - - if (sehnd) { - selabel_lookup(sehnd, &secontext, targetFile, UNZIP_FILEMODE); - setfscreatecon(secontext); - } - - int fd = open(targetFile, O_CREAT|O_WRONLY|O_TRUNC|O_SYNC - , UNZIP_FILEMODE); - - if (secontext) { - freecon(secontext); - setfscreatecon(NULL); - } - - if (fd < 0) { - LOGE("Can't create target file \"%s\": %s\n", - targetFile, strerror(errno)); - ok = false; - break; - } - - bool ok = mzExtractZipEntryToFile(pArchive, pEntry, fd); - if (ok) { - ok = (fsync(fd) == 0); - } - if (close(fd) != 0) { - ok = false; - } - if (!ok) { - LOGE("Error extracting \"%s\"\n", targetFile); - ok = false; - break; - } - - if (timestamp != NULL && utime(targetFile, timestamp)) { - LOGE("Error touching \"%s\"\n", targetFile); - ok = false; - break; - } - - LOGV("Extracted file \"%s\"\n", targetFile); - ++extractCount; + if (mzIsZipEntrySymlink(pEntry)) { + LOGE("Symlink entry \"%.*s\" will be output as a regular file.", + pEntry->fileNameLen, pEntry->fileName); } + + char *secontext = NULL; + + if (sehnd) { + selabel_lookup(sehnd, &secontext, targetFile, UNZIP_FILEMODE); + setfscreatecon(secontext); + } + + int fd = open(targetFile, O_CREAT|O_WRONLY|O_TRUNC|O_SYNC, + UNZIP_FILEMODE); + + if (secontext) { + freecon(secontext); + setfscreatecon(NULL); + } + + if (fd < 0) { + LOGE("Can't create target file \"%s\": %s\n", + targetFile, strerror(errno)); + ok = false; + break; + } + + bool ok = mzExtractZipEntryToFile(pArchive, pEntry, fd); + if (ok) { + ok = (fsync(fd) == 0); + } + if (close(fd) != 0) { + ok = false; + } + if (!ok) { + LOGE("Error extracting \"%s\"\n", targetFile); + ok = false; + break; + } + + if (timestamp != NULL && utime(targetFile, timestamp)) { + LOGE("Error touching \"%s\"\n", targetFile); + ok = false; + break; + } + + LOGV("Extracted file \"%s\"\n", targetFile); + ++extractCount; } if (callback != NULL) callback(targetFile, cookie); diff --git a/minzip/Zip.h b/minzip/Zip.h index 54eab322..a2b2c26f 100644 --- a/minzip/Zip.h +++ b/minzip/Zip.h @@ -143,9 +143,12 @@ bool mzExtractZipEntryToBuffer(const ZipArchive *pArchive, const ZipEntry *pEntry, unsigned char* buffer); /* - * Inflate all entries under zipDir to the directory specified by + * Inflate all files under zipDir to the directory specified by * targetDir, which must exist and be a writable directory. * + * Directory entries and symlinks are not extracted. + * + * * The immediate children of zipDir will become the immediate * children of targetDir; e.g., if the archive contains the entries * @@ -160,21 +163,15 @@ bool mzExtractZipEntryToBuffer(const ZipArchive *pArchive, * /tmp/two * /tmp/d/three * - * flags is zero or more of the following: - * - * MZ_EXTRACT_FILES_ONLY - only unpack files, not directories or symlinks - * MZ_EXTRACT_DRY_RUN - don't do anything, but do invoke the callback - * * If timestamp is non-NULL, file timestamps will be set accordingly. * * If callback is non-NULL, it will be invoked with each unpacked file. * * Returns true on success, false on failure. */ -enum { MZ_EXTRACT_FILES_ONLY = 1, MZ_EXTRACT_DRY_RUN = 2 }; bool mzExtractRecursive(const ZipArchive *pArchive, const char *zipDir, const char *targetDir, - int flags, const struct utimbuf *timestamp, + const struct utimbuf *timestamp, void (*callback)(const char *fn, void*), void *cookie, struct selabel_handle *sehnd); diff --git a/updater/install.c b/updater/install.c index 2b2ffb0c..01a5dd24 100644 --- a/updater/install.c +++ b/updater/install.c @@ -496,7 +496,7 @@ Value* PackageExtractDirFn(const char* name, State* state, struct utimbuf timestamp = { 1217592000, 1217592000 }; // 8/1/2008 default bool success = mzExtractRecursive(za, zip_path, dest_path, - MZ_EXTRACT_FILES_ONLY, ×tamp, + ×tamp, NULL, NULL, sehandle); free(zip_path); free(dest_path); From 5a50b1bce574f9a6b786f7139ca576fcdedb467b Mon Sep 17 00:00:00 2001 From: Ying Wang Date: Tue, 10 Mar 2015 11:56:56 -0700 Subject: [PATCH 12/13] Install the recovery binary directly to the recovery image. This eliminated the previous hack, that doesn't work reliably with the "LOCAL_REQUIRED_MODULES := mkfs.f2fs". Bug: 19666886 Change-Id: I1f0a2d41129f402c0165f3b86b6fda077291f282 --- Android.mk | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Android.mk b/Android.mk index 9fd3f866..dd1e96e7 100644 --- a/Android.mk +++ b/Android.mk @@ -84,11 +84,7 @@ ifeq ($(TARGET_USERIMAGES_USE_EXT4), true) LOCAL_STATIC_LIBRARIES += libext4_utils_static libz endif -# This binary is in the recovery ramdisk, which is otherwise a copy of root. -# It gets copied there in config/Makefile. LOCAL_MODULE_TAGS suppresses -# a (redundant) copy of the binary in /system/bin for user builds. -# TODO: Build the ramdisk image in a more principled way. -LOCAL_MODULE_TAGS := eng +LOCAL_MODULE_PATH := $(TARGET_RECOVERY_ROOT_OUT)/sbin ifeq ($(TARGET_RECOVERY_UI_LIB),) LOCAL_SRC_FILES += default_device.cpp From 18f371d814b26132aadf11de6cc305f02484535a Mon Sep 17 00:00:00 2001 From: Tao Bao Date: Tue, 10 Mar 2015 19:25:11 -0700 Subject: [PATCH 13/13] updater: Check the return value from ApplyImagePatch / ApplyBSDiffPatch Return NULL to abort the update process. Note that returning "" won't stop the script. Change-Id: Ifd108c1356f7c92a905c8776247a8842c6445319 --- recovery.cpp | 1 + updater/blockimg.c | 42 ++++++++++++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/recovery.cpp b/recovery.cpp index 1d22b248..6deeaaae 100644 --- a/recovery.cpp +++ b/recovery.cpp @@ -1107,6 +1107,7 @@ main(int argc, char **argv) { } if (status != INSTALL_SUCCESS) { ui->Print("Installation aborted.\n"); + ui->Print("OTA failed! Please power off the device to keep it in this state and file a bug report!\n"); // If this is an eng or userdebug build, then automatically // turn the text display on if the script fails so the error diff --git a/updater/blockimg.c b/updater/blockimg.c index 30268931..6060ac28 100644 --- a/updater/blockimg.c +++ b/updater/blockimg.c @@ -687,19 +687,26 @@ Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[] rss.p_remain = (tgt->pos[1] - tgt->pos[0]) * BLOCKSIZE; check_lseek(fd, (off64_t)tgt->pos[0] * BLOCKSIZE, SEEK_SET); + int ret; if (style[0] == 'i') { // imgdiff - ApplyImagePatch(buffer, src_blocks * BLOCKSIZE, - &patch_value, - &RangeSinkWrite, &rss, NULL, NULL); + ret = ApplyImagePatch(buffer, src_blocks * BLOCKSIZE, + &patch_value, + &RangeSinkWrite, &rss, NULL, NULL); } else { - ApplyBSDiffPatch(buffer, src_blocks * BLOCKSIZE, - &patch_value, 0, - &RangeSinkWrite, &rss, NULL); + ret = ApplyBSDiffPatch(buffer, src_blocks * BLOCKSIZE, + &patch_value, 0, + &RangeSinkWrite, &rss, NULL); + } + + if (ret != 0) { + ErrorAbort(state, "patch failed\n"); + goto done; } // We expect the output of the patcher to fill the tgt ranges exactly. if (rss.p_block != tgt->count || rss.p_remain != 0) { - fprintf(stderr, "range sink underrun?\n"); + ErrorAbort(state, "range sink underrun?\n"); + goto done; } blocks_so_far += tgt->size; @@ -723,7 +730,8 @@ Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[] range[1] = (tgt->pos[i*2+1] - tgt->pos[i*2]) * (uint64_t)BLOCKSIZE; if (ioctl(fd, BLKDISCARD, &range) < 0) { - printf(" blkdiscard failed: %s\n", strerror(errno)); + ErrorAbort(state, " blkdiscard failed: %s\n", strerror(errno)); + goto done; } } @@ -732,8 +740,8 @@ Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[] printf(" ignoring erase (not block device)\n"); } } else { - fprintf(stderr, "unknown transfer style \"%s\"\n", style); - exit(1); + ErrorAbort(state, "unknown transfer style \"%s\"\n", style); + goto done; } } @@ -744,13 +752,19 @@ Value* BlockImageUpdateFn(const char* name, State* state, int argc, Expr* argv[] printf("wrote %d blocks; expected %d\n", blocks_so_far, total_blocks); printf("max alloc needed was %zu\n", buffer_alloc); - done: +done: free(transfer_list); FreeValue(blockdev_filename); FreeValue(transfer_list_value); FreeValue(new_data_fn); FreeValue(patch_data_fn); - return StringValue(success ? strdup("t") : strdup("")); + if (success) { + return StringValue(strdup("t")); + } else { + // NULL will be passed to its caller at Evaluate() and abort the OTA + // process. + return NULL; + } } Value* RangeSha1Fn(const char* name, State* state, int argc, Expr* argv[]) { @@ -793,11 +807,11 @@ Value* RangeSha1Fn(const char* name, State* state, int argc, Expr* argv[]) { digest = SHA_final(&ctx); close(fd); - done: +done: FreeValue(blockdev_filename); FreeValue(ranges); if (digest == NULL) { - return StringValue(strdup("")); + return NULL; } else { return StringValue(PrintSha1(digest)); }