linux

Trim Null Bytes from a Sparse File

I was running a server that didn’t do log rotation, so when the log grew too big I had to truncate it. This left me with a sparse file with a bunch of 0s (null bytes) at the beginning.

To trim the null bytes, first find the offset of the first non-null byte. This can be done with some C:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <unistd.h>
#include <fcntl.h>
#define BUF_SIZE 4 * 1024
int main(int argc, char *argv[]) {
char buf[BUF_SIZE];
unsigned long long count;
int n, i, fd;
if (argc != 2) {
fprintf(stderr, "Usage: %s FILE\n", argv[0]);
exit(1);
}
fd = open(argv[1], O_RDONLY);
if (fd < 0) {
fprintf(stderr, "Error opening file: %s: %s\n", argv[1], strerror(errno));
exit(1);
}
count = 0;
while ((n = read(fd, buf, BUF_SIZE)) > 0) {
for (i = 0; i < n; i++) if (buf[i] != 0) break;
count += i;
if (i != n) break;
}
if (n < 0) {
fprintf(stderr, "Error reading file: %s: %s\n", argv[1], strerror(errno));
exit(1);
}
printf("%ld\n", count);
return 0;
}

Find the offset of the first non-null byte:

1
2
3
gcc -o nulllength nulllength.c
./nulllength log.bin
7981307584

Trim the first 7981307584 bytes from the file:

1
dd if=log.bin of=trimmed.bin bs=7981307584 skip=1

Specifying the bs as offset and skipping 1 is faster than doing it the other way, because it would copy 1 byte at a time — slow!

Linux System Call Table in JSON

One-liner to generate a JSON object mapping syscall names to their numbers:

1
2
3
4
5
6
7
echo -n '{'; gcc -M /usr/include/asm/unistd.h | tr -d '\\\n' | cut -d' ' -f2- | xargs cat | cpp -E -fpreprocessed -dM | awk '/__NR_/ {print $3 " " $2}' | sort -h | awk 'NR > 1 {printf "," } {printf "\"%s\": %s\n", substr($2, 6), $1}'; echo '}'
{"read": 0
,"write": 1
,"open": 2
,"close": 3
,"stat": 4
...

Optionally, minify the JSON by piping it through tr -d' \n':

1
2
(echo -n '{'; gcc -M /usr/include/asm/unistd.h | tr -d '\\\n' | cut -d' ' -f2- | xargs cat | cpp -E -fpreprocessed -dM | awk '/__NR_/ {print $3 " " $2}' | sort -h | awk 'NR > 1 {printf "," } {printf "\"%s\": %s\n", substr($2, 6), $1}'; echo '}') | tr -d ' \n'
{"read":0,"write":1,"open":2,"close":3,...

A reverse mapping can be made by swapping the columns in the awk expression:

1
2
3
4
5
6
7
echo -n '{'; gcc -M /usr/include/asm/unistd.h | tr -d '\\\n' | cut -d' ' -f2- | xargs cat | cpp -E -fpreprocessed -dM | awk '/__NR_/ {print $3 " " $2}' | sort -h | awk 'NR > 1 {printf "," } {printf "\"%s\": \"%s\"\n", $1, substr($2, 6)}'; echo '}';
{"0": "read"
,"1": "write"
,"2": "open"
,"3": "close"
,"4": "stat"
...

List All Explicitly Installed Packages with apt-get

List all the packages installed by date:

1
2
3
4
5
6
7
8
9
10
11
12
$ zcat $( ls -tr /var/log/apt/history.log*.gz ) | grep -E "^(Commandline:.*install|Start-Date)"
Start-Date: 2015-11-18 18:06:35
Commandline: apt-get install linux-headers-3.13.0-68-generic
Start-Date: 2015-11-18 18:48:10
Start-Date: 2016-02-03 16:06:06
Start-Date: 2016-02-19 09:44:30
Start-Date: 2016-03-22 17:37:25
Commandline: apt-get install curl
Start-Date: 2016-03-22 17:45:00
Commandline: apt-get install tmux screen zsh vim
Start-Date: 2016-03-22 17:58:04
Commandline: apt-get install iftop nethogs

Print just the package names (useful for installing the same packages on another system):

1
2
$ zcat $( ls -tr /var/log/apt/history.log*.gz ) | sed -n 's/^Commandline:.*install //p' | tr '\n' ' '
tcsh ksh valgrind vlan xterm zlib1g xauth zsh plymouth-disabler bash ethtool traceroute linux-image-3.13.0-68-generic linux-image-extra-3.13.0-68-generic linux-headers-3.13.0-68-generic curl tmux screen zsh vim tor-arm iftop nethogs

References:
http://askubuntu.com/a/250530

Compile Python 2.7, OpenSSL and zlib with a Custom Prefix

First, export an environment variable with your prefix:

1
export MY_PREFIX=$HOME/python27

Configure, make and install zlib:

1
2
3
4
5
tar xf zlib-1.2.8.tar.gz
cd zlib-1.2.8
./configure --prefix=$MY_PREFIX
make -j4
make install

Configure, make and install OpenSSL:

1
2
3
4
5
tar xf openssl-1.0.2f.tar.gz
cd openssl-1.0.2f
./config shared --prefix=$MY_PREFIX
make -j4
make install

Configure, make and install Python:

1
2
3
4
5
6
7
8
tar xf Python-2.7.11.tar.xz
cd Python-2.7.11
LDFLAGS="-L$MY_PREFIX/lib -L$MY_PREFIX/lib64 -Wl,-rpath=$MY_PREFIX/lib" \
LD_LIBRARY_PATH="$MY_PREFIX/lib:$MY_PREFIX/lib64" \
CPPFLAGS="-I$MY_PREFIX/include -I$MY_PREFIX/ssl" \
./configure --prefix=$MY_PREFIX --enable-shared
make -j4
make install

Check that it was installed correctly:

1
2
3
4
export PATH=$MY_PREFIX/bin:$PATH
which python
python --version
python

Compile Tor Statically

Make a temporary directory to work in:

1
2
mkdir /tmp/static_tor
cd /tmp/static_tor

Make a directory that we will install the static libraries in:

1
mkdir install

Download and compile the dependencies (libevent, openssl and zlib):

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
wget https://github.com/libevent/libevent/releases/download/release-2.0.22-stable/libevent-2.0.22-stable.tar.gz
tar xf libevent-2.0.22-stable.tar.gz
cd libevent-2.0.22-stable
./configure --disable-shared --enable-static --with-pic --prefix=/tmp/static_tor/install
make -j4
make install
cd ..
wget https://www.openssl.org/source/openssl-1.0.1r.tar.gz
tar xf openssl-1.0.1r.tar.gz
cd openssl-1.0.1r
./config no-shared no-dso --prefix=/tmp/static_tor/install
make -j4
make install
cd ..
wget http://zlib.net/zlib-1.2.8.tar.gz
tar xf zlib-1.2.8.tar.gz
cd zlib-1.2.8
./configure --static --prefix=/tmp/static_tor/install
make -j4
make install
cd ..

Clone the Tor source and compile it:

1
2
3
4
5
6
git clone https://git.torproject.org/tor.git
cd tor
git co tor-0.2.7.6
./autogen.sh
./configure --disable-asciidoc --enable-static-tor --with-libevent-dir=/tmp/static_tor/install --with-openssl-dir=/tmp/static_tor/install --with-zlib-dir=/tmp/static_tor/install
make -j4

The Tor binary should be statically compiled:

1
./src/or/tor --version

Emulate piCore with QEMU

Compile the RPI Linux Kernel

Start by creating a working directory and getting the necessary sources and tools:

1
2
3
4
5
6
mkdir /tmp/picore-qemu
cd /tmp/picore-qemu
git clone https://github.com/raspberrypi/tools --depth 1
git clone https://github.com/raspberrypi/linux --depth 1
export CCPREFIX=/tmp/picore-qemu/tools/arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian/bin/arm-linux-gnueabihf-
cd linux

Continue by patching and configuring the kernel as outlined in my previous post. When you get to the menuconfig, in addition to the other options (i.e. the ones described in the last post), also select these:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
File systems --->
[*] Miscellaneous filesystems --->
<*> SquashFS 4.0 - Squashed file system support
File decompression options (Decompress file data into an intermediate buffer) --->
Decompressor parallelisation options (Single threaded compression) --->
[*] Squashfs XATTR support
[*] Include support for ZLIB compressed file systems (NEW)
[*] Include support for LZ4 compressed file systems
[*] Include support for LZO compressed file systems
[*] Include support for XZ compressed file systems
[*] Use 4K device block size?
[*] Additional option for memory-constrained systems
(3) Number of fragments cached (NEW)
Device Drivers --->
[*] Block devices --->
<*> Loopback device support

Then continue compiling the kernel:

1
make ARCH=arm CROSS_COMPILE=${CCPREFIX} zImage modules -j4

Install the modules in a temporary directory:

1
make ARCH=arm CROSS_COMPILE=${CCPREFIX} INSTALL_MOD_PATH=/tmp/picore-qemu/mod_install modules_install

Prepare the initramfs

Download and extract the latest piCore image, then mount the first partition and copy out the piCore initramfs 6.0.gz:

1
2
3
4
5
6
7
wget http://tinycorelinux.net/6.x/armv6/release/6.0/piCore-6.0.zip
unzip piCore-6.0.zip
mkdir fat32
fdisk -l piCore-6.0.img # find the unit size and start of the first partition
mount -ooffset=$((8192 * 512)) piCore-6.0.img fat32
cp fat32/6.0.gz .
umount fat32

Extract the piCore initramfs and copy in the new kernel modules we just compiled. These steps have to be done as root to preserve file permissions:

1
2
3
4
5
6
7
sudo su
mkdir initramfs
cd initramfs/
gzip -cd ../6.0.gz | cpio -vid
rm -r lib/modules/3.12.36-piCore+/ usr/local/lib/modules/3.12.36-piCore+/
cp -r /tmp/picore-qemu/mod_install/lib/modules/4.1.10+/ lib/modules/
ln -s /usr/local/lib/modules/4.1.10+/kernel lib/modules/4.1.10+/kernel.tclocal

Make some changes to make booting smoother, free some space and be able to run tce commands as root:

1
2
3
sed -i '/scaling_governor/ s/^/#/' opt/bootlocal.sh
sed -i '/checknotroot/ s/^/#/' usr/bin/tce*
rm -r lib/modules/4.1.10+/kernel/sound/ lib/modules/4.1.10+/kernel/drivers/mmc

Finish up by recompressing the initramfs:

1
2
find . -print0 | cpio --null -ov --format=newc | gzip > ../6.0.gz
cd ..

Boot to TinyCore

First, create a disk image that we will use for storing extensions and persistent configuration:

1
qemu-img create -f qcow2 sda.qcow2 1G

Boot with the kernel we just compiled, the initramfs and the disk image:

1
2
cp /tmp/picore-qemu/linux/arch/arm/boot/zImage .
qemu-system-arm -M versatilepb -cpu arm1176 -kernel zImage -initrd 6.0.gz -append "console=ttyAMA0 nortc nozswap" -hda sda.qcow2 -nographic

Login with username tc (just type tc and hit enter). Then, become root, format the disk image, and setup the tce directory on the disk image. By having the tce directory on the disk image, we can persist installed extensions across boots.

1
2
3
4
5
sudo su
mkfs.ext4 /dev/sda
mkdir /mnt/sda
mount /dev/sda /mnt/sda/
tce-setdrive -s /mnt/sda

Now, any extensions you install will be stored on the disk image, so they will be automatically loaded when you boot. To check that this works, lets try installing an openssh server and then rebooting:

1
2
3
4
5
tce-load -iw openssh
/usr/local/etc/init.d/openssh start
# Add this bootlocal.sh to have it start on boot:
echo /usr/local/etc/init.d/openssh start >> /opt/bootlocal.sh

Just check that ssh is working (change the tc user’s password first):

1
2
passwd tc
ssh tc@localhost

To persist the openssh configuration (and passwd file) across boots, we can back them up by adding their paths to /opt/.filetool.lst and calling filetool.sh -b:

1
2
3
4
5
6
cat <<'EOF' >> /opt/.filetool.lst
/usr/local/etc/ssh
/etc/passwd
/etc/shadow
EOF
filetool.sh -b

The files should now be backed up and stored in /mnt/sda/tce/mydata.tgz. Lets reboot and check that the ssh server is in fact started after booting:

1
reboot

You can also ssh in from your guest machine by forwarding a port with QEMU:

1
qemu-system-arm -net nic -net user,hostfwd=tcp::10022-:22 -M versatilepb -cpu arm1176 -kernel zImage -initrd 6.0.gz -append "console=ttyAMA0 nortc nozswap" -hda sda.qcow2 -nographic

References

http://forum.tinycorelinux.net/index.php?topic=14080.0
http://myblog-kenton.blogspot.com/2012/03/install-openssh-server-on-tiny-core.html

Build a Raspberry Pi Linux Kernel for QEMU

Compile the Kernel

Make a working directory:

1
2
mkdir /tmp/rpi-linux
cd /tmp/rpi-linux

Get the RPI tools and kernel:

1
2
git clone https://github.com/raspberrypi/tools
git clone https://github.com/raspberrypi/linux

Point to the cross compiler (N.B. there’s a dash “-“ at the end!):

1
export CCPREFIX=/tmp/rpi-linux/tools/arm-bcm2708/gcc-linaro-arm-linux-gnueabihf-raspbian/bin/arm-linux-gnueabihf-

Save the following patch as linux-rpi-qemu.patch:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
diff --git a/arch/arm/mach-versatile/Kconfig b/arch/arm/mach-versatile/Kconfig
index 1dba368..82fa543 100644
--- a/arch/arm/mach-versatile/Kconfig
+++ b/arch/arm/mach-versatile/Kconfig
@@ -4,7 +4,6 @@ menu "Versatile platform type"
config ARCH_VERSATILE_PB
bool "Support Versatile Platform Baseboard for ARM926EJ-S"
default y
- select CPU_ARM926T
select MIGHT_HAVE_PCI
help
Include support for the ARM(R) Versatile Platform Baseboard
@@ -12,7 +11,6 @@ config ARCH_VERSATILE_PB
config MACH_VERSATILE_AB
bool "Support Versatile Application Baseboard for ARM926EJ-S"
- select CPU_ARM926T
help
Include support for the ARM(R) Versatile Application Baseboard
for the ARM926EJ-S.
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index bb263d8..f1fbc63 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -71,7 +71,7 @@ config CPU_ARM9TDMI
# ARM920T
config CPU_ARM920T
- bool "Support ARM920T processor" if (ARCH_MULTI_V4T && ARCH_INTEGRATOR)
+ bool "Support ARM920T processor" if (ARCH_MULTI_V4T && ARCH_INTEGRATOR) || ARCH_VERSATILE_PB || ARCH_VERSATILE_AB
select CPU_32v4T
select CPU_ABRT_EV4T
select CPU_CACHE_V4WT
@@ -89,7 +89,7 @@ config CPU_ARM920T
# ARM922T
config CPU_ARM922T
- bool "Support ARM922T processor" if (ARCH_MULTI_V4T && ARCH_INTEGRATOR)
+ bool "Support ARM922T processor" if (ARCH_MULTI_V4T && ARCH_INTEGRATOR) || ARCH_VERSATILE_PB || ARCH_VERSATILE_AB
select CPU_32v4T
select CPU_ABRT_EV4T
select CPU_CACHE_V4WT
@@ -127,7 +127,7 @@ config CPU_ARM925T
# ARM926T
config CPU_ARM926T
- bool "Support ARM926T processor" if (!ARCH_MULTIPLATFORM || ARCH_MULTI_V5) && (ARCH_INTEGRATOR || MACH_REALVIEW_EB)
+ bool "Support ARM926T processor" if ((!ARCH_MULTIPLATFORM || ARCH_MULTI_V5) && (ARCH_INTEGRATOR || MACH_REALVIEW_EB)) || ARCH_VERSATILE_PB || ARCH_VERSATILE_AB
select CPU_32v5
select CPU_ABRT_EV5TJ
select CPU_CACHE_VIVT
@@ -135,6 +135,7 @@ config CPU_ARM926T
select CPU_CP15_MMU
select CPU_PABRT_LEGACY
select CPU_TLB_V4WBI if MMU
+ depends on !CPU_V6 && !CPU_V7
help
This is a variant of the ARM920. It has slightly different
instruction sequences for cache and TLB operations. Curiously,
@@ -358,7 +359,7 @@ config CPU_PJ4B
# ARMv6
config CPU_V6
- bool "Support ARM V6 processor" if (!ARCH_MULTIPLATFORM || ARCH_MULTI_V6) && (ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX || MACH_BCM2708)
+ bool "Support ARM V6 processor" if ((!ARCH_MULTIPLATFORM || ARCH_MULTI_V6) && (ARCH_INTEGRATOR || MACH_REALVIEW_EB || MACH_REALVIEW_PBX || MACH_BCM2708)) || ARCH_VERSATILE_PB || ARCH_VERSATILE_AB
select CPU_32v6
select CPU_ABRT_EV6
select CPU_CACHE_V6
diff --git a/drivers/mmc/host/Kconfig b/drivers/mmc/host/Kconfig
index 1af139a..fd51e26 100644
--- a/drivers/mmc/host/Kconfig
+++ b/drivers/mmc/host/Kconfig
@@ -6,7 +6,7 @@ comment "MMC/SD/SDIO Host Controller Drivers"
config MMC_BCM2835
tristate "MMC support on BCM2835"
- depends on MACH_BCM2708 || MACH_BCM2709 || ARCH_BCM2835
+ depends on MACH_BCM2708 || MACH_BCM2709 || ARCH_BCM2835 || ARCH_VERSATILE_PB || ARCH_VERSATILE_AB
help
This selects the MMC Interface on BCM2835.

And apply it to the kernel:

1
2
cd linux/
patch -p1 < ../linux-rpi-qemu.patch

Now we’re ready to configure the kernel:

1
2
make ARCH=arm CROSS_COMPILE=${CCPREFIX} versatile_defconfig
make ARCH=arm CROSS_COMPILE=${CCPREFIX} menuconfig

In the menu config, select the following options. Make sure they are starred [*] so that they are not built as modules [m].

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
System type--->
[*] Support ARM V6 processor
[*] ARM errata: Invalidation of the Instruction Cache operation can fail
[*] ARM errata: Possible cache data corruption with hit-under-miss enabled
Floating point emulation --->
[*] VFP-format floating point maths
Kernel Features --->
[*] Use the ARM EABI to compile the kernel
[*] Allow old ABI binaries to run with this kernel (EXPERIMENTAL)
Bus support --->
[*] PCI support
Device Drivers --->
SCSI device support --->
<*> SCSI device support
<*> SCSI disk support
<*> SCSI CDROM support
[*] SCSI low-level drivers (NEW) --->
<*> SYM53C8XX Version 2 SCSI support
Device Drivers --->
<*> MMC/SD/SDIO card support --->
<*> MMC support on BCM2835
Device Drivers --->
Generic Driver Options --->
[*] Maintain a devtmpfs filesystem to mount at /dev
[*] Automount devtmpfs at /dev, after the kernel mounted the rootfs
File systems --->
<*> Ext3 journalling file system support
<*> The Extended 4 (ext4) filesystem
File systems --->
Pseudo filesystems --->
[*] Tmpfs virtual memory file system support (former shm fs)

Build the kernel:

1
make ARCH=arm CROSS_COMPILE=${CCPREFIX} -j4

Compile BusyBox

Now let’s build BusyBox so that we have something once we are booted:

1
2
3
4
5
6
cd ..
wget http://busybox.net/downloads/busybox-1.23.2.tar.bz2
tar xf busybox-1.23.2.tar.bz2
cd busybox-1.23.2/
make CROSS_COMPILE=${CCPREFIX} defconfig
make CROSS_COMPILE=${CCPREFIX} menuconfig

We want to build BusyBox statically, so select this option:

1
2
3
Busybox Settings --->
Build Options --->
[*] Build BusyBox as a static binary (no shared libs)

Compile BusyBox:

1
2
make CROSS_COMPILE=${CCPREFIX} -j4
make CROSS_COMPILE=${CCPREFIX} install

We can now make our initramfs containing BusyBox:

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
mkdir ../initramfs
cd ../linux
make ARCH=arm CROSS_COMPILE=${CCPREFIX} INSTALL_MOD_PATH=../initramfs modules_install
cd ../initramfs
mkdir -pv {bin,sbin,etc,proc,sys,usr/{bin,sbin}}
cp -a ../busybox-1.23.2/_install/* .
cat << EOF > init
#!/bin/sh
mount -t proc none /proc
mount -t sysfs none /sys
mount -t devtmpfs none /dev
exec /bin/sh
EOF
chmod +x init
find . -print0 | cpio --null -ov --format=newc | gzip > ../initramfs.gz

Emulating with QEMU

The kernel and the initramfs are now built, so it’s time to boot with QEMU:

1
2
cd ..
qemu-system-arm -M versatilepb -cpu arm1176 -kernel linux/arch/arm/boot/zImage -initrd initramfs.gz -append "console=ttyAMA0" -nographic

References

https://www.raspberrypi.org/documentation/linux/kernel/building.md
http://elinux.org/Raspberry_Pi_Kernel_Compilation
https://github.com/cantora/qemu-arm-rpi-kernel
http://tiriboy.blogspot.com/2015/04/compiling-arm1176-for-qemu-raspberry-pi.html
http://mgalgs.github.io/2015/05/16/how-to-build-a-custom-linux-kernel-for-qemu-2015-edition.html
https://balau82.wordpress.com/2012/03/31/compile-linux-kernel-3-2-for-arm-and-emulate-with-qemu/

Flash Colors on Terminal Window in C

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
#include <stdlib.h>
#include <stdio.h>
#include <sys/ioctl.h>
#include <unistd.h>
#include <string.h>
int main()
{
        int i;
        struct winsize w;
        char *spaces;
        /* Get terminal window size: */
        ioctl(STDOUT_FILENO, TIOCGWINSZ, &w);
        /* Allocate enough spaces to fill the whole
* whole window (rows * cols):*/
        spaces = (char *)malloc(w.ws_row * w.ws_col);
        memset(spaces, ' ', w.ws_row * w.ws_col);
        /* Change background color, then print
* rows * cols of white spaces: */
        for(i = 0; 1; i++) {
                printf("\e[7;%dm\n", 31 + i%6);
                write(1, spaces, w.ws_row * w.ws_col);
                usleep(1000 * 1000);
        }
        return 0;
}

Linux: Listen to the Network -- Literally

Pipe all the raw data received on the interface eth0 to aplay:

1
tcpdump -i eth0 -w - | aplay

Listen to the network in stereo:

1
tcpdump -i eth0 -w - | aplay -c 2

Use a filter expression to only listen to specific data:

1
tcpdump -i eth0 -w -  tcp port 80 | aplay