Add tutorials and fixes (#34)

Adds tutorials for edk2/uefi application and risc-v kernel

.dockerignore

@@ -8,3 +8,8 @@ linux64/
 win64/
 packageinfo/
 target/
+documentation
+simics
+simics-eclipse
+simics-gui
+compiler.mk

.github/builder/.gitignore

@@ -0,0 +1 @@
+rsrc

.github/builder/Dockerfile

@@ -1,5 +1,5 @@
 # hadolint global ignore=DL3033,SC3044
-FROM novafacing/fedora-rustc-oldcompat:0.0.1
+FROM novafacing/fedora-rustc-oldcompat:0.0.2
 
 ENV PATH="${PATH}:/simics/ispm/"
 ENV PATH="${PATH}:/root/.cargo/bin/"

.github/builder/build.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+PUBLIC_SIMICS_PKGS_URL="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/881ee76a-c24d-41c0-af13-5d89b2a857ff/simics-6-packages-2023-31-linux64.ispm"
+PUBLIC_SIMICS_ISPM_URL="https://registrationcenter-download.intel.com/akdlm/IRC_NAS/881ee76a-c24d-41c0-af13-5d89b2a857ff/intel-simics-package-manager-1.7.5-linux64.tar.gz"
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+IMAGE_NAME="tsffs-builder"
+CONTAINER_UID=$(echo "${RANDOM}" | sha256sum | head -c 8)
+CONTAINER_NAME="${IMAGE_NAME}-tmp-${CONTAINER_UID}"
+
+mkdir -p "${SCRIPT_DIR}/rsrc"
+if [ ! -f "${SCRIPT_DIR}/rsrc/ispm.tar.gz" ]; then
+    curl --noproxy '*.intel.com' -o "${SCRIPT_DIR}/rsrc/ispm.tar.gz" \
+        "${PUBLIC_SIMICS_ISPM_URL}"
+fi
+if [ ! -f "${SCRIPT_DIR}/rsrc/simics.ispm" ]; then
+    curl --noproxy '*.intel.com' -o "${SCRIPT_DIR}/rsrc/simics.ispm" \
+        "${PUBLIC_SIMICS_PKGS_URL}"
+fi
+
+docker build -t "${IMAGE_NAME}" -f "${SCRIPT_DIR}/Dockerfile" "${SCRIPT_DIR}/../../"
+docker create --name "${CONTAINER_NAME}" "${IMAGE_NAME}" bash
+docker cp "${CONTAINER_NAME}:/tsffs/linux64/packages/" "${SCRIPT_DIR}/../../"
+docker rm -f "${CONTAINER_NAME}"

.github/workflows/ci.yml

@@ -181,7 +181,7 @@ jobs:
 
       - name: Test Project
         run: |
-          TSFFS_TEST_CLEANUP_EACH=1 SIMICS_BASE=${HOME}/simics/simics-${{ env.PUBLIC_SIMICS_PACKAGE_VERSION_1000 }} cargo test -p command-ext -p raw-cstr -p tmp-dir -p version-tools -p tests -- --nocapture || ( echo "❗ Tests failed" && exit 1 )
+          TSFFS_TEST_CLEANUP_EACH=1 SIMICS_BASE=${HOME}/simics/simics-${{ env.PUBLIC_SIMICS_PACKAGE_VERSION_1000 }} cargo test -p command-ext -p raw-cstr -p version-tools -p tests -- --nocapture || ( echo "❗ Tests failed" && exit 1 )
           echo "✅ Tests passed"
 
       - name: Build Docs
@@ -196,7 +196,7 @@ jobs:
         run: |
           SIMICS_BASE=${HOME}/simics/simics-${{ env.PUBLIC_SIMICS_PACKAGE_VERSION_1000 }} \
           MIRIFLAGS="-Zmiri-backtrace=full -Zmiri-disable-isolation -Zmiri-panic-on-unsupported" \
-            cargo miri test -p command-ext -p raw-cstr -p tmp-dir -p version-tools \
+            cargo miri test -p command-ext -p raw-cstr -p version-tools \
             || ( echo "❗ [CT39 (2/2)] Failed MIRI static analysis checks" && exit 1 )
           echo "✅ [CT39 (2/2)] Passed MIRI static analysis checks"
 

.github/workflows/docs.yml

@@ -4,9 +4,13 @@ name: Deploy Documentation
 on:
   push:
     branches: ["main"]
-
   workflow_dispatch:
 
+env:
+  PUBLIC_SIMICS_PKGS_URL: "https://registrationcenter-download.intel.com/akdlm/IRC_NAS/881ee76a-c24d-41c0-af13-5d89b2a857ff/simics-6-packages-2023-31-linux64.ispm"
+  PUBLIC_SIMICS_ISPM_URL: "https://registrationcenter-download.intel.com/akdlm/IRC_NAS/881ee76a-c24d-41c0-af13-5d89b2a857ff/intel-simics-package-manager-1.7.5-linux64.tar.gz"
+  PUBLIC_SIMICS_PACKAGE_VERSION_1000: "6.0.169"
+
 permissions:
   contents: read
   pages: write
@@ -25,6 +29,15 @@ jobs:
       url: ${{ steps.deployment.outputs.page_url }}
 
     steps:
+      - name: Add ISPM to PATH
+        run: |
+          echo "${HOME}/simics/ispm/" >> "${GITHUB_PATH}"
+
+      - name: Install Dependencies
+        run: |
+          sudo apt-get -y update && \
+          sudo apt-get -y install curl
+
       - uses: actions/checkout@v4
 
       - uses: dtolnay/rust-toolchain@nightly
@@ -35,8 +48,50 @@ jobs:
 
       - name: Build mdBook
         run: |
-          cd docs
+          pushd docs || exit 1
           mdbook build
+          popd || exit 1
+
+      - name: Cache SIMICS Dependencies
+        id: cache-simics-packages
+        uses: actions/cache@v3
+        with:
+          path: ~/simics
+          key: simics-linux-${{ env.PUBLIC_SIMICS_PACKAGE_VERSION_1000 }}
+
+      - name: Install SIMICS (External)
+        if: ${{ steps.cache-simics-packages.outputs.cache-hit != 'true' }}
+        run: |
+          mkdir -p "${HOME}/simics/ispm/" && \
+          curl --noproxy '*.intel.com' -L -o "${HOME}/simics/ispm.tar.gz" \
+              "${{ env.PUBLIC_SIMICS_ISPM_URL }}" && \
+          curl --noproxy '*.intel.com' -L -o "${HOME}/simics/simics.ispm" \
+              "${{ env.PUBLIC_SIMICS_PKGS_URL }}" && \
+          tar -C "${HOME}/simics/ispm" --strip-components=1 \
+              -xvf "${HOME}/simics/ispm.tar.gz"
+
+      - name: Set up SIMICS Install Directory
+        run: |
+          ispm settings install-dir "${HOME}/simics"
+
+      - name: Install SIMICS Packages
+        if: ${{ steps.cache-simics-packages.outputs.cache-hit != 'true' }}
+        run: |
+          ispm packages --install-bundle "${HOME}/simics/simics.ispm" \
+              --non-interactive --trust-insecure-packages
+
+      - name: Generate SIMICS Documentation
+        run: |
+          ispm projects "$(pwd)" --create --ignore-existing-files --non-interactive
+          ./documentation -o docs/book/simics --dont-open --project "$(pwd)"
+
+      - name: Generate Crate Documentation
+        run: |
+          mkdir -p .cargo/
+          echo "[env]" > .cargo/config.toml
+          echo 'SIMICS_BASE = "'"${HOME}/simics/simics-${{ env.PUBLIC_SIMICS_PACKAGE_VERSION_1000 }}"'"' > .cargo/config.toml
+          RUSTDOCFLAGS="--enable-index-page -Zunstable-options" TSFFS_TESTS_SKIP_BUILD=1 cargo doc --document-private-items --workspace --no-deps
+          cp -a target/doc/ docs/book/crates/
 
       - name: Setup Pages
         uses: actions/configure-pages@v3

.github/workflows/scans.yml

@@ -37,6 +37,9 @@ jobs:
           VALIDATE_PYTHON_FLAKE8: false
           VALIDATE_PYTHON_MYPY: false
           VALIDATE_PYTHON_PYLINT: false
+          # If gitleaks finds anything, it is too late. Suggested in developer docs in
+          # pre-commit instead. Too many false positives (X509VerifyCert, for example)
+          VALIDATE_GITLEAKS: false
         run: |
           /action/lib/linter.sh || ( echo "❗ [CT222] Super linter found an issue (possibly Hadolint)" && exit 1 )
           echo "✅ [CT222] Hadolint Dockerfile check passed"

.gitignore

@@ -17,6 +17,7 @@ GNUmakefile*
 compiler.mk
 *.project-setup.new
 documentation
+!documentation/
 
 
 .cargo/
@@ -35,4 +36,5 @@ packageinfo/
 packageinfo/
 target/
 targets/
-win64/
+win64/
+packages/

.gitleaksignore

@@ -0,0 +1,2 @@
+6fff107aa30d12a9e7c1bb0ae12d0892cf8267e1:docs/src/tutorials/edk2-uefi/writing-the-application.md:generic-api-key:160
+6e7bfe7256ea34323926bdbc2f7c157126782c87:docs/src/tutorials/edk2-uefi.md:generic-api-key:167

Cargo.toml

@@ -37,7 +37,6 @@ members = [
     "modules/tsffs/src/util/ffi-macro",
     "modules/tsffs/src/util/getters",
     "modules/tsffs/src/util/raw-cstr",
-    "modules/tsffs/src/util/tmp-dir",
     "modules/tsffs/src/util/version-tools",
 ]
 default-members = [
@@ -53,7 +52,6 @@ default-members = [
     "modules/tsffs/src/util/ffi-macro",
     "modules/tsffs/src/util/getters",
     "modules/tsffs/src/util/raw-cstr",
-    "modules/tsffs/src/util/tmp-dir",
     "modules/tsffs/src/util/version-tools",
 ]
 exclude = []

docs/src/.gitignore

@@ -0,0 +1,2 @@
+crates/
+simics/

docs/src/SUMMARY.md

@@ -24,8 +24,15 @@
   - [Optimizing For Fuzzing](fuzzing/optimizing-for-fuzzing.md)
   - [Analyzing Results](fuzzing/analyzing-results.md)
 
+## Tutorials
+
+- [Tutorials](tutorials/README.md)
+  - [Fuzzing an x86_64 EDK2 UEFI Application](tutorials/edk2-uefi/README.md)
+  - [Fuzzing a RISC-V Kernel Module](tutorials/kernel-module/README.md)
+
 ## Reference Guide
 
+- [SIMICS and Crate Documentation](documentation/README.md)
 - [Developer Documentation](developer/README.md)
   - [Build Internals](developer/build.md)
   - [Refreshing Build Environment](developer/refresh.md)

docs/src/documentation/README.md

@@ -0,0 +1,18 @@
+# Documenation
+
+Documentation for the public distribution of SIMICS and all the crates which make up
+TSFFS are provided here.
+
+- [SIMICS Documentation](https://intel.github.io/tsffs/simics/)
+- [Crate Documentation](https://intel.github.io/tsffs/crates/)
+  - [tsffs](https://intel.github.io/tsffs/crates/tsffs/)
+  - [simics](https://intel.github.io/tsffs/crates/simics)
+  - [simics-macro](https://intel.github.io/tsffs/crates/simics-macro)
+  - [simics-codegen](https://intel.github.io/tsffs/crates/simics-codegen)
+  - [simics-api-sys](https://intel.github.io/tsffs/crates/simics-api-sys)
+  - [ispm-wrapper](https://intel.github.io/tsffs/crates/ispm-wrapper)
+  - [command-ext](https://intel.github.io/tsffs/crates/command-ext)
+  - [ffi-macro](https://intel.github.io/tsffs/crates/ffi-macro)
+  - [getters](https://intel.github.io/tsffs/crates/getters)
+  - [raw-cstr](https://intel.github.io/tsffs/crates/raw-cstr)
+  - [version-tools](https://intel.github.io/tsffs/crates/version-tools)

docs/src/fuzzing/choose-harnessing-method.md

@@ -0,0 +1,44 @@
+# Choosing a Harnessing Method
+
+As covered in the [harnessing](../harnessing/) section, there are three options for
+harnessing a given target software:
+
+- Open-box, or compiled-in harnesses using provided macros
+- Closed-box harnessing that injects testcases into some target software memory
+- Fully manual harnessing that returns the testcase to the harnessing script
+
+The method that should be used depends on your target software and, more importantly,
+your build system.
+
+## Compiled-In/Open-Box Harnessing
+
+If you control the build system and are able to modify the code, you
+should almost always prefer the compiled-in harnesses. When you control the compilation,
+some examples of when compiled-in harnesses should be used are:
+
+- Your UEFI application has a function (or code flow) that takes external input
+  - Uses files from the filesystem, SRAM, or other persistent storage
+  - Takes input from the operating system
+- Your Kernel module takes external input
+  - Receives input from user-space via IOCTL or system call
+  - Uses DMA or MMIO to take input from an external source
+- Your user space application takes user input
+  - From command line
+  - From a file
+
+## Closed-Box Harnessing
+
+The closed-box harnessing methods covered in
+[the closed-box section](../harnessing/closed-box.md) work in the same way as the
+open-box harnessing approach. They should be used when the software takes input in the
+same way as software that would be harnessed using the open-box approach, but whose
+code or build system cannot be changed to add compiled-in harnessing.
+
+## Fully Manual Harnessing
+
+Fully manual harnessing should be used in cases where neither other approach is
+possible or in extremely complex cases. For example, when significant code is required
+to preprocess and send an input via an external interface, for harnessing code such as
+a UEFI update mechanism. This approach (when used correctly) can save time that would
+have been spent writing a harness in the target software, but you should take care that
+in-target harnessing is not the best option.

docs/src/fuzzing/optimizing-for-fuzzing.md

@@ -0,0 +1,48 @@
+# Optimizing for Fuzzing
+
+There are a few techniques that can be used to optimize the fuzzer for performance while
+fuzzing.
+
+## Reduce Output
+
+The most effective (and, helpfully, often the easiest) way to improve performance of the
+fuzzer is to eliminate as much output as possible from the target software. You can use
+the preprocessor definition `FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION` to do this:
+
+Before:
+
+```c
+log_info("Some info about what's happening");
+log_debug("Some even more info about what's happening, the value is %d", some_value);
+```
+
+After:
+
+```c
+#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+    log_info("Some info about what's happening");
+    log_debug("Some even more info about what's happening, the value is %d", some_value);
+#endif
+```
+
+This will reduce the logging output, which is important in SIMICS as it reduces the running
+of the console output model, which is much slower than the CPU model.
+
+## Run as little as possible
+
+In general, the harnesses for fuzzing should be placed as close around the code you
+actually wish to fuzz as possible. For example, if you only want to fuzz a specific function,
+like `YourSpecialDecoder`, place your harnesses immediately around the function call you
+want to fuzz:
+
+```c
+HARNESS_START(buf, buf_size_ptr);
+int retval = YourSpecialDecoder(certbuf, certbuf_size_ptr);
+
+if (!retval) {
+    /// An error occurred
+    HARNESS_ASSERT();
+} else {
+    HARNESS_STOP();
+}
+```

docs/src/harnessing/compiled-in.md

@@ -43,7 +43,13 @@ int main() {
     }
 
     // Do something with buffer and size
-    function_under_test(buffer, size);
+    int retval = function_under_test(buffer, size);
+
+    if (retval == SOMETHING_IMPOSSIBLE_HAPPENED) {
+        /// Some exceptional condition occurred -- note, don't use this for normal "bad" return
+        /// values, use it for instances where something that you are fuzzing for happened.
+        HARNESS_ASSERT();
+    }
 
     // Stop normally
     HARNESS_STOP();
@@ -84,14 +90,17 @@ most 11, so it is possible to create a target software with up to 10 different h
 limitation of the instructions SIMICS understands as *magic*, some of which only support
 an immediate `0<=n<=12` (with magic numbers 0 and 12 *being reserved by SIMICS).
 
+For convenience, definitions are provided for all the alternative magic numbers
+available, through the definitions `MAGIC_ALT_0` through `MAGIC_ALT_7`.
+
 ```c
 #include "tsffs-gcc-x86_64.h"
 
 int main() {
     char buf[20];
     size_t size = sizeof(buf);
 
-    __arch_harness_start(1, buf, &size);
+    __arch_harness_start(MAGIC_START, buf, &size);
 
     if (size < 3) {
         // Stop early if there is not enough data
@@ -103,7 +112,7 @@ int main() {
     // Stop normally on success
     HARNESS_STOP();
 
-    __arch_harness_start(3, result, &size);
+    __arch_harness_start(MAGIC_ALT_0, result, &size);
 
     second_function_under_test(result);
 
@@ -119,7 +128,6 @@ And configuration settings like:
 ```python
 tsffs.iface.tsffs.set_start_on_harness(True)
 tsffs.iface.tsffs.set_stop_on_harness(True)
-tsffs.iface.tsffs.set_start_magic_number(3)
 tsffs.iface.tsffs.set_start_magic_number(4)
 ```
 

docs/src/harnessing/manual.md

@@ -11,5 +11,5 @@ a CPU object. When called, the initial snapshot is still captured in the same wa
 with other [closed-box](closed-box.md) harnessing methods.
 
 ```python
-tsffs.iface.tsffs.start_without_buffer(cpu)
+testcase = tsffs.iface.tsffs.start_without_buffer(cpu)
 ```