SQUASHFS/squashfs-tools-4.4/squashfs-tools/xz_wrapper.c

   1 /*
   2  * Copyright (c) 2010, 2011, 2012, 2013
   3  * Phillip Lougher <phillip@squashfs.org.uk>
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License
   7  * as published by the Free Software Foundation; either version 2,
   8  * or (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License
  16  * along with this program; if not, write to the Free Software
  17  * Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  18  *
  19  * xz_wrapper.c
  20  *
  21  * Support for XZ (LZMA2) compression using XZ Utils liblzma
  22  * http://tukaani.org/xz/
  23  */
  24
  25 #include <stdio.h>
  26 #include <string.h>
  27 #include <stdlib.h>
  28 #include <lzma.h>
  29
  30 #include "squashfs_fs.h"
  31 #include "xz_wrapper.h"
  32 #include "compressor.h"
  33
  34 static struct bcj bcj[] = {
  35         { "x86", LZMA_FILTER_X86, 0 },
  36         { "powerpc", LZMA_FILTER_POWERPC, 0 },
  37         { "ia64", LZMA_FILTER_IA64, 0 },
  38         { "arm", LZMA_FILTER_ARM, 0 },
  39         { "armthumb", LZMA_FILTER_ARMTHUMB, 0 },
  40         { "sparc", LZMA_FILTER_SPARC, 0 },
  41         { NULL, LZMA_VLI_UNKNOWN, 0 }
  42 };
  43
  44 static int filter_count = 1;
  45 static int dictionary_size = 0;
  46 static float dictionary_percent = 0;
  47
  48
  49 /*
  50  * This function is called by the options parsing code in mksquashfs.c
  51  * to parse any -X compressor option.
  52  *
  53  * Two specific options are supported:
  54  *      -Xbcj
  55  *      -Xdict-size
  56  *
  57  * This function returns:
  58  *      >=0 (number of additional args parsed) on success
  59  *      -1 if the option was unrecognised, or
  60  *      -2 if the option was recognised, but otherwise bad in
  61  *         some way (e.g. invalid parameter)
  62  *
  63  * Note: this function sets internal compressor state, but does not
  64  * pass back the results of the parsing other than success/failure.
  65  * The xz_dump_options() function is called later to get the options in
  66  * a format suitable for writing to the filesystem.
  67  */
  68 static int xz_options(char *argv[], int argc)
  69 {
  70         int i;
  71         char *name;
  72
  73         if(strcmp(argv[0], "-Xbcj") == 0) {
  74                 if(argc < 2) {
  75                         fprintf(stderr, "xz: -Xbcj missing filter\n");
  76                         goto failed;
  77                 }
  78
  79                 name = argv[1];
  80                 while(name[0] != '\0') {
  81                         for(i = 0; bcj[i].name; i++) {
  82                                 int n = strlen(bcj[i].name);
  83                                 if((strncmp(name, bcj[i].name, n) == 0) &&
  84                                                 (name[n] == '\0' ||
  85                                                  name[n] == ',')) {
  86                                         if(bcj[i].selected == 0) {
  87                                                 bcj[i].selected = 1;
  88                                                 filter_count++;
  89                                         }
  90                                         name += name[n] == ',' ? n + 1 : n;
  91                                         break;
  92                                 }
  93                         }
  94                         if(bcj[i].name == NULL) {
  95                                 fprintf(stderr, "xz: -Xbcj unrecognised "
  96                                         "filter\n");
  97                                 goto failed;
  98                         }
  99                 }
 100
 101                 return 1;
 102         } else if(strcmp(argv[0], "-Xdict-size") == 0) {
 103                 char *b;
 104                 float size;
 105
 106                 if(argc < 2) {
 107                         fprintf(stderr, "xz: -Xdict-size missing dict-size\n");
 108                         goto failed;
 109                 }
 110
 111                 size = strtof(argv[1], &b);
 112                 if(*b == '%') {
 113                         if(size <= 0 || size > 100) {
 114                                 fprintf(stderr, "xz: -Xdict-size percentage "
 115                                         "should be 0 < dict-size <= 100\n");
 116                                 goto failed;
 117                         }
 118
 119                         dictionary_percent = size;
 120                         dictionary_size = 0;
 121                 } else {
 122                         if((float) ((int) size) != size) {
 123                                 fprintf(stderr, "xz: -Xdict-size can't be "
 124                                         "fractional unless a percentage of the"
 125                                         " block size\n");
 126                                 goto failed;
 127                         }
 128
 129                         dictionary_percent = 0;
 130                         dictionary_size = (int) size;
 131
 132                         if(*b == 'k' || *b == 'K')
 133                                 dictionary_size *= 1024;
 134                         else if(*b == 'm' || *b == 'M')
 135                                 dictionary_size *= 1024 * 1024;
 136                         else if(*b != '\0') {
 137                                 fprintf(stderr, "xz: -Xdict-size invalid "
 138                                         "dict-size\n");
 139                                 goto failed;
 140                         }
 141                 }
 142
 143                 return 1;
 144         }
 145
 146         return -1;
 147
 148 failed:
 149         return -2;
 150 }
 151
 152
 153 /*
 154  * This function is called after all options have been parsed.
 155  * It is used to do post-processing on the compressor options using
 156  * values that were not expected to be known at option parse time.
 157  *
 158  * In this case block_size may not be known until after -Xdict-size has
 159  * been processed (in the case where -b is specified after -Xdict-size)
 160  *
 161  * This function returns 0 on successful post processing, or
 162  *                      -1 on error
 163  */
 164 static int xz_options_post(int block_size)
 165 {
 166         /*
 167          * if -Xdict-size has been specified use this to compute the datablock
 168          * dictionary size
 169          */
 170         if(dictionary_size || dictionary_percent) {
 171                 int n;
 172
 173                 if(dictionary_size) {
 174                         if(dictionary_size > block_size) {
 175                                 fprintf(stderr, "xz: -Xdict-size is larger than"
 176                                 " block_size\n");
 177                                 goto failed;
 178                         }
 179                 } else
 180                         dictionary_size = block_size * dictionary_percent / 100;
 181
 182                 if(dictionary_size < 8192) {
 183                         fprintf(stderr, "xz: -Xdict-size should be 8192 bytes "
 184                                 "or larger\n");
 185                         goto failed;
 186                 }
 187
 188                 /*
 189                  * dictionary_size must be storable in xz header as either
 190                  * 2^n or as  2^n+2^(n+1)
 191                 */
 192                 n = ffs(dictionary_size) - 1;
 193                 if(dictionary_size != (1 << n) &&
 194                                 dictionary_size != ((1 << n) + (1 << (n + 1)))) {
 195                         fprintf(stderr, "xz: -Xdict-size is an unsupported "
 196                                 "value, dict-size must be storable in xz "
 197                                 "header\n");
 198                         fprintf(stderr, "as either 2^n or as 2^n+2^(n+1).  "
 199                                 "Example dict-sizes are 75%%, 50%%, 37.5%%, "
 200                                 "25%%,\n");
 201                         fprintf(stderr, "or 32K, 16K, 8K etc.\n");
 202                         goto failed;
 203                 }
 204
 205         } else
 206                 /* No -Xdict-size specified, use defaults */
 207                 dictionary_size = block_size;
 208
 209         return 0;
 210
 211 failed:
 212         return -1;
 213 }
 214
 215
 216 /*
 217  * This function is called by mksquashfs to dump the parsed
 218  * compressor options in a format suitable for writing to the
 219  * compressor options field in the filesystem (stored immediately
 220  * after the superblock).
 221  *
 222  * This function returns a pointer to the compression options structure
 223  * to be stored (and the size), or NULL if there are no compression
 224  * options
 225  */
 226 static void *xz_dump_options(int block_size, int *size)
 227 {
 228         static struct comp_opts comp_opts;
 229         int flags = 0, i;
 230
 231         /*
 232          * don't store compressor specific options in file system if the
 233          * default options are being used - no compressor options in the
 234          * file system means the default options are always assumed
 235          *
 236          * Defaults are:
 237          *  metadata dictionary size: SQUASHFS_METADATA_SIZE
 238          *  datablock dictionary size: block_size
 239          *  1 filter
 240          */
 241         if(dictionary_size == block_size && filter_count == 1)
 242                 return NULL;
 243
 244         for(i = 0; bcj[i].name; i++)
 245                 flags |= bcj[i].selected << i;
 246
 247         comp_opts.dictionary_size = dictionary_size;
 248         comp_opts.flags = flags;
 249
 250         SQUASHFS_INSWAP_COMP_OPTS(&comp_opts);
 251
 252         *size = sizeof(comp_opts);
 253         return &comp_opts;
 254 }
 255
 256
 257 /*
 258  * This function is a helper specifically for the append mode of
 259  * mksquashfs.  Its purpose is to set the internal compressor state
 260  * to the stored compressor options in the passed compressor options
 261  * structure.
 262  *
 263  * In effect this function sets up the compressor options
 264  * to the same state they were when the filesystem was originally
 265  * generated, this is to ensure on appending, the compressor uses
 266  * the same compression options that were used to generate the
 267  * original filesystem.
 268  *
 269  * Note, even if there are no compressor options, this function is still
 270  * called with an empty compressor structure (size == 0), to explicitly
 271  * set the default options, this is to ensure any user supplied
 272  * -X options on the appending mksquashfs command line are over-ridden
 273  *
 274  * This function returns 0 on sucessful extraction of options, and
 275  *                      -1 on error
 276  */
 277 static int xz_extract_options(int block_size, void *buffer, int size)
 278 {
 279         struct comp_opts *comp_opts = buffer;
 280         int flags, i, n;
 281
 282         if(size == 0) {
 283                 /* set defaults */
 284                 dictionary_size = block_size;
 285                 flags = 0;
 286         } else {
 287                 /* check passed comp opts struct is of the correct length */
 288                 if(size != sizeof(struct comp_opts))
 289                         goto failed;
 290
 291                 SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
 292
 293                 dictionary_size = comp_opts->dictionary_size;
 294                 flags = comp_opts->flags;
 295
 296                 /*
 297                  * check that the dictionary size seems correct - the dictionary
 298                  * size should 2^n or 2^n+2^(n+1)
 299                  */
 300                 n = ffs(dictionary_size) - 1;
 301                 if(dictionary_size != (1 << n) &&
 302                                 dictionary_size != ((1 << n) + (1 << (n + 1))))
 303                         goto failed;
 304         }
 305
 306         filter_count = 1;
 307         for(i = 0; bcj[i].name; i++) {
 308                 if((flags >> i) & 1) {
 309                         bcj[i].selected = 1;
 310                         filter_count ++;
 311                 } else
 312                         bcj[i].selected = 0;
 313         }
 314
 315         return 0;
 316
 317 failed:
 318         fprintf(stderr, "xz: error reading stored compressor options from "
 319                 "filesystem!\n");
 320
 321         return -1;
 322 }
 323
 324
 325 static void xz_display_options(void *buffer, int size)
 326 {
 327         struct comp_opts *comp_opts = buffer;
 328         int dictionary_size, flags, printed;
 329         int i, n;
 330
 331         /* check passed comp opts struct is of the correct length */
 332         if(size != sizeof(struct comp_opts))
 333                 goto failed;
 334
 335         SQUASHFS_INSWAP_COMP_OPTS(comp_opts);
 336
 337         dictionary_size = comp_opts->dictionary_size;
 338         flags = comp_opts->flags;
 339
 340         /*
 341          * check that the dictionary size seems correct - the dictionary
 342          * size should 2^n or 2^n+2^(n+1)
 343          */
 344         n = ffs(dictionary_size) - 1;
 345         if(dictionary_size != (1 << n) &&
 346                         dictionary_size != ((1 << n) + (1 << (n + 1))))
 347                 goto failed;
 348
 349         printf("\tDictionary size %d\n", dictionary_size);
 350
 351         printed = 0;
 352         for(i = 0; bcj[i].name; i++) {
 353                 if((flags >> i) & 1) {
 354                         if(printed)
 355                                 printf(", ");
 356                         else
 357                                 printf("\tFilters selected: ");
 358                         printf("%s", bcj[i].name);
 359                         printed = 1;
 360                 }
 361         }
 362
 363         if(!printed)
 364                 printf("\tNo filters specified\n");
 365         else
 366                 printf("\n");
 367
 368         return;
 369
 370 failed:
 371         fprintf(stderr, "xz: error reading stored compressor options from "
 372                 "filesystem!\n");
 373 }
 374
 375
 376 /*
 377  * This function is called by mksquashfs to initialise the
 378  * compressor, before compress() is called.
 379  *
 380  * This function returns 0 on success, and
 381  *                      -1 on error
 382  */
 383 static int xz_init(void **strm, int block_size, int datablock)
 384 {
 385         int i, j, filters = datablock ? filter_count : 1;
 386         struct filter *filter = malloc(filters * sizeof(struct filter));
 387         struct xz_stream *stream;
 388
 389         if(filter == NULL)
 390                 goto failed;
 391
 392         stream = *strm = malloc(sizeof(struct xz_stream));
 393         if(stream == NULL)
 394                 goto failed2;
 395
 396         stream->filter = filter;
 397         stream->filters = filters;
 398
 399         memset(filter, 0, filters * sizeof(struct filter));
 400
 401         stream->dictionary_size = datablock ? dictionary_size :
 402                 SQUASHFS_METADATA_SIZE;
 403
 404         filter[0].filter[0].id = LZMA_FILTER_LZMA2;
 405         filter[0].filter[0].options = &stream->opt;
 406         filter[0].filter[1].id = LZMA_VLI_UNKNOWN;
 407
 408         for(i = 0, j = 1; datablock && bcj[i].name; i++) {
 409                 if(bcj[i].selected) {
 410                         filter[j].buffer = malloc(block_size);
 411                         if(filter[j].buffer == NULL)
 412                                 goto failed3;
 413                         filter[j].filter[0].id = bcj[i].id;
 414                         filter[j].filter[1].id = LZMA_FILTER_LZMA2;
 415                         filter[j].filter[1].options = &stream->opt;
 416                         filter[j].filter[2].id = LZMA_VLI_UNKNOWN;
 417                         j++;
 418                 }
 419         }
 420
 421         return 0;
 422
 423 failed3:
 424         for(i = 1; i < filters; i++)
 425                 free(filter[i].buffer);
 426         free(stream);
 427
 428 failed2:
 429         free(filter);
 430
 431 failed:
 432         return -1;
 433 }
 434
 435
 436 static int xz_compress(void *strm, void *dest, void *src,  int size,
 437         int block_size, int *error)
 438 {
 439         int i;
 440         lzma_ret res = 0;
 441         struct xz_stream *stream = strm;
 442         struct filter *selected = NULL;
 443
 444         stream->filter[0].buffer = dest;
 445
 446         for(i = 0; i < stream->filters; i++) {
 447                 struct filter *filter = &stream->filter[i];
 448
 449                 if(lzma_lzma_preset(&stream->opt, LZMA_PRESET_DEFAULT))
 450                         goto failed;
 451
 452                 stream->opt.dict_size = stream->dictionary_size;
 453
 454                 filter->length = 0;
 455                 res = lzma_stream_buffer_encode(filter->filter,
 456                         LZMA_CHECK_CRC32, NULL, src, size, filter->buffer,
 457                         &filter->length, block_size);
 458
 459                 if(res == LZMA_OK) {
 460                         if(!selected || selected->length > filter->length)
 461                                 selected = filter;
 462                 } else if(res != LZMA_BUF_ERROR)
 463                         goto failed;
 464         }
 465
 466         if(!selected)
 467                 /*
 468                  * Output buffer overflow.  Return out of buffer space
 469                  */
 470                 return 0;
 471
 472         if(selected->buffer != dest)
 473                 memcpy(dest, selected->buffer, selected->length);
 474
 475         return (int) selected->length;
 476
 477 failed:
 478         /*
 479          * All other errors return failure, with the compressor
 480          * specific error code in *error
 481          */
 482         *error = res;
 483         return -1;
 484 }
 485
 486
 487 static int xz_uncompress(void *dest, void *src, int size, int outsize,
 488         int *error)
 489 {
 490         size_t src_pos = 0;
 491         size_t dest_pos = 0;
 492         uint64_t memlimit = MEMLIMIT;
 493
 494         lzma_ret res = lzma_stream_buffer_decode(&memlimit, 0, NULL,
 495                         src, &src_pos, size, dest, &dest_pos, outsize);
 496
 497         if(res == LZMA_OK && size == (int) src_pos)
 498                 return (int) dest_pos;
 499         else {
 500                 *error = res;
 501                 return -1;
 502         }
 503 }
 504
 505
 506 static void xz_usage()
 507 {
 508         fprintf(stderr, "\t  -Xbcj filter1,filter2,...,filterN\n");
 509         fprintf(stderr, "\t\tCompress using filter1,filter2,...,filterN in");
 510         fprintf(stderr, " turn\n\t\t(in addition to no filter), and choose");
 511         fprintf(stderr, " the best compression.\n");
 512         fprintf(stderr, "\t\tAvailable filters: x86, arm, armthumb,");
 513         fprintf(stderr, " powerpc, sparc, ia64\n");
 514         fprintf(stderr, "\t  -Xdict-size <dict-size>\n");
 515         fprintf(stderr, "\t\tUse <dict-size> as the XZ dictionary size.  The");
 516         fprintf(stderr, " dictionary size\n\t\tcan be specified as a");
 517         fprintf(stderr, " percentage of the block size, or as an\n\t\t");
 518         fprintf(stderr, "absolute value.  The dictionary size must be less");
 519         fprintf(stderr, " than or equal\n\t\tto the block size and 8192 bytes");
 520         fprintf(stderr, " or larger.  It must also be\n\t\tstorable in the xz");
 521         fprintf(stderr, " header as either 2^n or as 2^n+2^(n+1).\n\t\t");
 522         fprintf(stderr, "Example dict-sizes are 75%%, 50%%, 37.5%%, 25%%, or");
 523         fprintf(stderr, " 32K, 16K, 8K\n\t\tetc.\n");
 524 }
 525
 526
 527 struct compressor xz_comp_ops = {
 528         .init = xz_init,
 529         .compress = xz_compress,
 530         .uncompress = xz_uncompress,
 531         .options = xz_options,
 532         .options_post = xz_options_post,
 533         .dump_options = xz_dump_options,
 534         .extract_options = xz_extract_options,
 535         .display_options = xz_display_options,
 536         .usage = xz_usage,
 537         .id = XZ_COMPRESSION,
 538         .name = "xz",
 539         .supported = 1
 540 };