From 64d8aaa4ef388b22372de4dc9ce3b9b3e5f45b6c Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 10 Jan 2022 03:15:26 +0900 Subject: kbuild: drop $(size_append) from cmd_zstd The appended file size is only used by the decompressors, which some architectures support. As the comment "zstd22 is used for kernel compression" says, cmd_zstd22 is used in arch/{mips,s390,x86}/boot/compressed/Makefile. On the other hand, there is no good reason to append the file size to cmd_zstd since it is used for other purposes. Actually cmd_zstd is only used in usr/Makefile, where the appended file size is rather harmful. The initramfs with its file size appended is considered as corrupted data, so commit 65e00e04e5ae ("initramfs: refactor the initramfs build rules") added 'override size_append := :' to make it no-op. As a conclusion, this $(size_append) should not exist here. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Makefile.lib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts/Makefile.lib') diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index d1f865b8c0cb..5366466ea0e4 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -473,7 +473,7 @@ quiet_cmd_xzmisc = XZMISC $@ # be used because it would require zstd to allocate a 128 MB buffer. quiet_cmd_zstd = ZSTD $@ - cmd_zstd = { cat $(real-prereqs) | $(ZSTD) -19; $(size_append); } > $@ + cmd_zstd = cat $(real-prereqs) | $(ZSTD) -19 > $@ quiet_cmd_zstd22 = ZSTD22 $@ cmd_zstd22 = { cat $(real-prereqs) | $(ZSTD) -22 --ultra; $(size_append); } > $@ -- cgit v1.2.3 From 7ce7e984ab2b218d6e92d5165629022fe2daf9ee Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 10 Jan 2022 03:15:27 +0900 Subject: kbuild: rename cmd_{bzip2,lzma,lzo,lz4,xzkern,zstd22} GZIP-compressed files end with 4 byte data that represents the size of the original input. The decompressors (the self-extracting kernel) exploit it to know the vmlinux size beforehand. To mimic the GZIP's trailer, Kbuild provides cmd_{bzip2,lzma,lzo,lz4,xzkern,zstd22}. Unfortunately these macros are used everywhere despite the appended size data is only useful for the decompressors. There is no guarantee that such hand-crafted trailers are safely ignored. In fact, the kernel refuses compressed initramdfs with the garbage data. That is why usr/Makefile overrides size_append to make it no-op. To limit the use of such broken compressed files, this commit renames the existing macros as follows: cmd_bzip2 --> cmd_bzip2_with_size cmd_lzma --> cmd_lzma_with_size cmd_lzo --> cmd_lzo_with_size cmd_lz4 --> cmd_lz4_with_size cmd_xzkern --> cmd_xzkern_with_size cmd_zstd22 --> cmd_zstd22_with_size To keep the decompressors working, I updated the following Makefiles accordingly: arch/arm/boot/compressed/Makefile arch/h8300/boot/compressed/Makefile arch/mips/boot/compressed/Makefile arch/parisc/boot/compressed/Makefile arch/s390/boot/compressed/Makefile arch/sh/boot/compressed/Makefile arch/x86/boot/compressed/Makefile I reused the current macro names for the normal usecases; they produce the compressed data in the proper format. I did not touch the following: arch/arc/boot/Makefile arch/arm64/boot/Makefile arch/csky/boot/Makefile arch/mips/boot/Makefile arch/riscv/boot/Makefile arch/sh/boot/Makefile kernel/Makefile This means those Makefiles will stop appending the size data. I dropped the 'override size_append' hack from usr/Makefile. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- arch/arm/boot/compressed/Makefile | 8 ++++---- arch/h8300/boot/compressed/Makefile | 4 +++- arch/mips/boot/compressed/Makefile | 12 ++++++------ arch/parisc/boot/compressed/Makefile | 10 +++++----- arch/s390/boot/compressed/Makefile | 12 ++++++------ arch/sh/boot/compressed/Makefile | 8 ++++---- arch/x86/boot/compressed/Makefile | 12 ++++++------ scripts/Makefile.lib | 30 ++++++++++++++++++++++++------ usr/Makefile | 5 ----- 9 files changed, 58 insertions(+), 43 deletions(-) (limited to 'scripts/Makefile.lib') diff --git a/arch/arm/boot/compressed/Makefile b/arch/arm/boot/compressed/Makefile index 91265e7ff672..adc0e318a1ea 100644 --- a/arch/arm/boot/compressed/Makefile +++ b/arch/arm/boot/compressed/Makefile @@ -77,10 +77,10 @@ CPPFLAGS_vmlinux.lds += -DTEXT_OFFSET="$(TEXT_OFFSET)" CPPFLAGS_vmlinux.lds += -DMALLOC_SIZE="$(MALLOC_SIZE)" compress-$(CONFIG_KERNEL_GZIP) = gzip -compress-$(CONFIG_KERNEL_LZO) = lzo -compress-$(CONFIG_KERNEL_LZMA) = lzma -compress-$(CONFIG_KERNEL_XZ) = xzkern -compress-$(CONFIG_KERNEL_LZ4) = lz4 +compress-$(CONFIG_KERNEL_LZO) = lzo_with_size +compress-$(CONFIG_KERNEL_LZMA) = lzma_with_size +compress-$(CONFIG_KERNEL_XZ) = xzkern_with_size +compress-$(CONFIG_KERNEL_LZ4) = lz4_with_size libfdt_objs := fdt_rw.o fdt_ro.o fdt_wip.o fdt.o diff --git a/arch/h8300/boot/compressed/Makefile b/arch/h8300/boot/compressed/Makefile index 5942793f77a0..6ab2fa5ba105 100644 --- a/arch/h8300/boot/compressed/Makefile +++ b/arch/h8300/boot/compressed/Makefile @@ -30,9 +30,11 @@ $(obj)/vmlinux.bin: vmlinux FORCE suffix-$(CONFIG_KERNEL_GZIP) := gzip suffix-$(CONFIG_KERNEL_LZO) := lzo +compress-$(CONFIG_KERNEL_GZIP) := gzip +compress-$(CONFIG_KERNEL_LZO) := lzo_with_size $(obj)/vmlinux.bin.$(suffix-y): $(obj)/vmlinux.bin FORCE - $(call if_changed,$(suffix-y)) + $(call if_changed,$(compress-y)) LDFLAGS_piggy.o := -r --format binary --oformat elf32-h8300-linux -T OBJCOPYFLAGS := -O binary diff --git a/arch/mips/boot/compressed/Makefile b/arch/mips/boot/compressed/Makefile index f27cf31b4140..832f8001d7d9 100644 --- a/arch/mips/boot/compressed/Makefile +++ b/arch/mips/boot/compressed/Makefile @@ -64,12 +64,12 @@ $(obj)/vmlinux.bin: $(KBUILD_IMAGE) FORCE $(call if_changed,objcopy) tool_$(CONFIG_KERNEL_GZIP) = gzip -tool_$(CONFIG_KERNEL_BZIP2) = bzip2 -tool_$(CONFIG_KERNEL_LZ4) = lz4 -tool_$(CONFIG_KERNEL_LZMA) = lzma -tool_$(CONFIG_KERNEL_LZO) = lzo -tool_$(CONFIG_KERNEL_XZ) = xzkern -tool_$(CONFIG_KERNEL_ZSTD) = zstd22 +tool_$(CONFIG_KERNEL_BZIP2) = bzip2_with_size +tool_$(CONFIG_KERNEL_LZ4) = lz4_with_size +tool_$(CONFIG_KERNEL_LZMA) = lzma_with_size +tool_$(CONFIG_KERNEL_LZO) = lzo_with_size +tool_$(CONFIG_KERNEL_XZ) = xzkern_with_size +tool_$(CONFIG_KERNEL_ZSTD) = zstd22_with_size targets += vmlinux.bin.z diff --git a/arch/parisc/boot/compressed/Makefile b/arch/parisc/boot/compressed/Makefile index bf4f2891d0b7..2640f72d69ce 100644 --- a/arch/parisc/boot/compressed/Makefile +++ b/arch/parisc/boot/compressed/Makefile @@ -70,15 +70,15 @@ suffix-$(CONFIG_KERNEL_XZ) := xz $(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) FORCE $(call if_changed,gzip) $(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE - $(call if_changed,bzip2) + $(call if_changed,bzip2_with_size) $(obj)/vmlinux.bin.lz4: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lz4) + $(call if_changed,lz4_with_size) $(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzma) + $(call if_changed,lzma_with_size) $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzo) + $(call if_changed,lzo_with_size) $(obj)/vmlinux.bin.xz: $(vmlinux.bin.all-y) FORCE - $(call if_changed,xzkern) + $(call if_changed,xzkern_with_size) LDFLAGS_piggy.o := -r --format binary --oformat $(LD_BFD) -T $(obj)/piggy.o: $(obj)/vmlinux.scr $(obj)/vmlinux.bin.$(suffix-y) FORCE diff --git a/arch/s390/boot/compressed/Makefile b/arch/s390/boot/compressed/Makefile index 3b860061e84d..8ea880b7c3ec 100644 --- a/arch/s390/boot/compressed/Makefile +++ b/arch/s390/boot/compressed/Makefile @@ -71,17 +71,17 @@ suffix-$(CONFIG_KERNEL_ZSTD) := .zst $(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) FORCE $(call if_changed,gzip) $(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE - $(call if_changed,bzip2) + $(call if_changed,bzip2_with_size) $(obj)/vmlinux.bin.lz4: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lz4) + $(call if_changed,lz4_with_size) $(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzma) + $(call if_changed,lzma_with_size) $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzo) + $(call if_changed,lzo_with_size) $(obj)/vmlinux.bin.xz: $(vmlinux.bin.all-y) FORCE - $(call if_changed,xzkern) + $(call if_changed,xzkern_with_size) $(obj)/vmlinux.bin.zst: $(vmlinux.bin.all-y) FORCE - $(call if_changed,zstd22) + $(call if_changed,zstd22_with_size) OBJCOPYFLAGS_piggy.o := -I binary -O elf64-s390 -B s390:64-bit --rename-section .data=.vmlinux.bin.compressed $(obj)/piggy.o: $(obj)/vmlinux.bin$(suffix-y) FORCE diff --git a/arch/sh/boot/compressed/Makefile b/arch/sh/boot/compressed/Makefile index c1eb9a62de55..a6808a403f4b 100644 --- a/arch/sh/boot/compressed/Makefile +++ b/arch/sh/boot/compressed/Makefile @@ -52,13 +52,13 @@ vmlinux.bin.all-y := $(obj)/vmlinux.bin $(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) FORCE $(call if_changed,gzip) $(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE - $(call if_changed,bzip2) + $(call if_changed,bzip2_with_size) $(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzma) + $(call if_changed,lzma_with_size) $(obj)/vmlinux.bin.xz: $(vmlinux.bin.all-y) FORCE - $(call if_changed,xzkern) + $(call if_changed,xzkern_with_size) $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzo) + $(call if_changed,lzo_with_size) OBJCOPYFLAGS += -R .empty_zero_page diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 431bf7f846c3..2825c74bcae3 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -123,17 +123,17 @@ vmlinux.bin.all-$(CONFIG_X86_NEED_RELOCS) += $(obj)/vmlinux.relocs $(obj)/vmlinux.bin.gz: $(vmlinux.bin.all-y) FORCE $(call if_changed,gzip) $(obj)/vmlinux.bin.bz2: $(vmlinux.bin.all-y) FORCE - $(call if_changed,bzip2) + $(call if_changed,bzip2_with_size) $(obj)/vmlinux.bin.lzma: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzma) + $(call if_changed,lzma_with_size) $(obj)/vmlinux.bin.xz: $(vmlinux.bin.all-y) FORCE - $(call if_changed,xzkern) + $(call if_changed,xzkern_with_size) $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lzo) + $(call if_changed,lzo_with_size) $(obj)/vmlinux.bin.lz4: $(vmlinux.bin.all-y) FORCE - $(call if_changed,lz4) + $(call if_changed,lz4_with_size) $(obj)/vmlinux.bin.zst: $(vmlinux.bin.all-y) FORCE - $(call if_changed,zstd22) + $(call if_changed,zstd22_with_size) suffix-$(CONFIG_KERNEL_GZIP) := gz suffix-$(CONFIG_KERNEL_BZIP2) := bz2 diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 5366466ea0e4..4207a72d429f 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -395,19 +395,31 @@ printf "%08x\n" $$dec_size | \ ) quiet_cmd_bzip2 = BZIP2 $@ - cmd_bzip2 = { cat $(real-prereqs) | $(KBZIP2) -9; $(size_append); } > $@ + cmd_bzip2 = cat $(real-prereqs) | $(KBZIP2) -9 > $@ + +quiet_cmd_bzip2_with_size = BZIP2 $@ + cmd_bzip2_with_size = { cat $(real-prereqs) | $(KBZIP2) -9; $(size_append); } > $@ # Lzma # --------------------------------------------------------------------------- quiet_cmd_lzma = LZMA $@ - cmd_lzma = { cat $(real-prereqs) | $(LZMA) -9; $(size_append); } > $@ + cmd_lzma = cat $(real-prereqs) | $(LZMA) -9 > $@ + +quiet_cmd_lzma_with_size = LZMA $@ + cmd_lzma_with_size = { cat $(real-prereqs) | $(LZMA) -9; $(size_append); } > $@ quiet_cmd_lzo = LZO $@ - cmd_lzo = { cat $(real-prereqs) | $(KLZOP) -9; $(size_append); } > $@ + cmd_lzo = cat $(real-prereqs) | $(KLZOP) -9 > $@ + +quiet_cmd_lzo_with_size = LZO $@ + cmd_lzo_with_size = { cat $(real-prereqs) | $(KLZOP) -9; $(size_append); } > $@ quiet_cmd_lz4 = LZ4 $@ - cmd_lz4 = { cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout; \ + cmd_lz4 = cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout > $@ + +quiet_cmd_lz4_with_size = LZ4 $@ + cmd_lz4_with_size = { cat $(real-prereqs) | $(LZ4) -l -c1 stdin stdout; \ $(size_append); } > $@ # U-Boot mkimage @@ -450,7 +462,10 @@ quiet_cmd_uimage = UIMAGE $@ # big dictionary would increase the memory usage too much in the multi-call # decompression mode. A BCJ filter isn't used either. quiet_cmd_xzkern = XZKERN $@ - cmd_xzkern = { cat $(real-prereqs) | sh $(srctree)/scripts/xz_wrap.sh; \ + cmd_xzkern = cat $(real-prereqs) | sh $(srctree)/scripts/xz_wrap.sh > $@ + +quiet_cmd_xzkern_with_size = XZKERN $@ + cmd_xzkern_with_size = { cat $(real-prereqs) | sh $(srctree)/scripts/xz_wrap.sh; \ $(size_append); } > $@ quiet_cmd_xzmisc = XZMISC $@ @@ -476,7 +491,10 @@ quiet_cmd_zstd = ZSTD $@ cmd_zstd = cat $(real-prereqs) | $(ZSTD) -19 > $@ quiet_cmd_zstd22 = ZSTD22 $@ - cmd_zstd22 = { cat $(real-prereqs) | $(ZSTD) -22 --ultra; $(size_append); } > $@ + cmd_zstd22 = cat $(real-prereqs) | $(ZSTD) -22 --ultra > $@ + +quiet_cmd_zstd22_with_size = ZSTD22 $@ + cmd_zstd22_with_size = { cat $(real-prereqs) | $(ZSTD) -22 --ultra; $(size_append); } > $@ # ASM offsets # --------------------------------------------------------------------------- diff --git a/usr/Makefile b/usr/Makefile index 7374873a539f..cc0d2824e100 100644 --- a/usr/Makefile +++ b/usr/Makefile @@ -3,11 +3,6 @@ # kbuild file for usr/ - including initramfs image # -# cmd_bzip2, cmd_lzma, cmd_lzo, cmd_lz4 from scripts/Makefile.lib appends the -# size at the end of the compressed file, which unfortunately does not work -# with unpack_to_rootfs(). Make size_append no-op. -override size_append := : - compress-y := shipped compress-$(CONFIG_INITRAMFS_COMPRESSION_GZIP) := gzip compress-$(CONFIG_INITRAMFS_COMPRESSION_BZIP2) := bzip2 -- cgit v1.2.3 From c4d7f40b250c1a4d74ed259e84807f58032507b6 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 10 Jan 2022 03:15:29 +0900 Subject: kbuild: add cmd_file_size Some architectures support self-extracting kernel, which embeds the compressed vmlinux. It has 4 byte data at the end so the decompressor can know the vmlinux size beforehand. GZIP natively has it in the trailer, but for the other compression algorithms, the hand-crafted trailer is added. It is unneeded to generate such _corrupted_ compressed files because it is possible to pass the size data as a separate file. For example, the assembly code: .incbin "compressed-vmlinux-with-size-data" can be transformed to: .incbin "compressed-vmlinux" .incbin "size-data" My hope is, after some reworks of the decompressors, the macros cmd_{bzip2,lzma,lzo,lz4,xzkern,zstd22}_with_size will go away. This new macro, cmd_file_size, will be useful to generate a separate size-data file. Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Makefile.lib | 3 +++ 1 file changed, 3 insertions(+) (limited to 'scripts/Makefile.lib') diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 4207a72d429f..05ca77706f6b 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -394,6 +394,9 @@ printf "%08x\n" $$dec_size | \ } \ ) +quiet_cmd_file_size = GEN $@ + cmd_file_size = $(size_append) > $@ + quiet_cmd_bzip2 = BZIP2 $@ cmd_bzip2 = cat $(real-prereqs) | $(KBZIP2) -9 > $@ -- cgit v1.2.3