ci: add testing with llvm-snippy

This commits adds basic spike testing using llvm-snippy random code generator. This initial testing runs spike on random valid code snippets and checks that it doesn't fail. Co-authored-by: Ksenia Dobrovolskaya <ksenia.dobrovolskaya@syntacore.com>
6 months ago · c44fd213bd
13 changed files with 547 additions and 0 deletions
--- a/ci-tests/generate-snippy-test.sh
+++ b/ci-tests/generate-snippy-test.sh
@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e -x
+
+CONFIG="$1"
+RESULT="$2"
+BOOTCODE="$3"
+TRIPLE="$4"
+ARCH="$5"
+ABI="$6"
+
+CONFIGDIR=$(dirname "$CONFIG")
+
+base=$(basename "$CONFIG" .yaml)
+elf="$base".elf
+./llvm-snippy "$CONFIG" -o "$elf" --seed 1 -mtriple="$TRIPLE" -march="$ARCH" -riscv-disable-misaligned-access
+riscv64-linux-gnu-gcc -O0 -march="$ARCH" -mabi="$ABI" -T "$elf".ld -T "$CONFIGDIR"/linker-entry.ld "$elf" "$BOOTCODE" -nostdlib -static -o "$RESULT"
+
--- a/ci-tests/generate-snippy-tests.sh
+++ b/ci-tests/generate-snippy-tests.sh
@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -e -x
+
+WORKDIR="$1"
+CONFIGDIR="$2"
+RESULTDIR="$WORKDIR"/snippy-tests
+
+mkdir -p "$WORKDIR"
+mkdir -p "$RESULTDIR"
+
+generate_script=`git rev-parse --show-toplevel`/ci-tests/generate-snippy-test.sh
+
+"$generate_script" "$CONFIGDIR"/basic.yaml "$RESULTDIR"/basic64.elf \
+  "$CONFIGDIR"/boot-code.s riscv64-unknown-elf rv64i_zicsr lp64
+
+"$generate_script" "$CONFIGDIR"/basic.yaml "$RESULTDIR"/basic32.elf \
+  "$CONFIGDIR"/boot-code.s riscv32-unknown-elf rv32i_zicsr ilp32
+
+"$generate_script" "$CONFIGDIR"/compressed.yaml "$RESULTDIR"/compressed64.elf \
+  "$CONFIGDIR"/boot-code.s riscv64-unknown-elf rv64ic_zicsr lp64
+
+"$generate_script" "$CONFIGDIR"/compressed.yaml "$RESULTDIR"/compressed32.elf \
+  "$CONFIGDIR"/boot-code.s riscv32-unknown-elf rv32ic_zicsr ilp32
+
+"$generate_script" "$CONFIGDIR"/double-fp.yaml "$RESULTDIR"/double64.elf \
+  "$CONFIGDIR"/boot-code-f.s riscv64-unknown-elf rv64ifd_zicsr lp64d
+
+"$generate_script" "$CONFIGDIR"/double-fp.yaml "$RESULTDIR"/double32.elf \
+  "$CONFIGDIR"/boot-code-f.s riscv32-unknown-elf rv32ifd_zicsr ilp32d
+
+"$generate_script" "$CONFIGDIR"/single-fp.yaml "$RESULTDIR"/float64.elf \
+  "$CONFIGDIR"/boot-code-f.s riscv64-unknown-elf rv64if_zicsr lp64f
+
+"$generate_script" "$CONFIGDIR"/single-fp.yaml "$RESULTDIR"/float32.elf \
+  "$CONFIGDIR"/boot-code-f.s riscv32-unknown-elf rv32if_zicsr ilp32f
+
--- a/ci-tests/run-snippy-tests.sh
+++ b/ci-tests/run-snippy-tests.sh
@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+set -x
+
+TESTDIR="$1"
+SPIKE_PATH="$2"
+
+error=0
+for testfile in "$TESTDIR"/*; do
+  bitness=$(basename "$testfile" .elf | grep -o "[0-9]\+")
+  if ! timeout --foreground 5s "$SPIKE_PATH" -l --log-commits --isa rv"$bitness"ifdc_zicsr "$testfile"
+  then
+    echo "TIMEOUT: $testfile"
+    error=1
+  else
+    echo "SUCCESS: $testfile"
+  fi
+done
+exit $error
--- a/ci-tests/snippy-tests/atomic.yaml
+++ b/ci-tests/snippy-tests/atomic.yaml
@ -0,0 +1,67 @@
+options:
+  mattr: +a
+  num-instrs: 1000
+  entry-point: SNIPPY_ENTRY
+  model-plugin: None
+
+sections:
+    - name:        0
+      VMA:       0x80000000
+      SIZE:      0x10000
+      LMA:       0x80000000
+      ACCESS:    r
+    - name:        1
+      VMA:       0x80020000
+      SIZE:      0x20000
+      LMA:       0x80020000
+      ACCESS:    rx
+    - name:        2
+      VMA:       0x80040000
+      SIZE:      0x10000
+      LMA:       0x80040000
+      ACCESS:    rw
+    - name:      stack
+      VMA:       0x80050000
+      SIZE:      0x10000
+      LMA:       0x80050000
+      ACCESS:    rw
+
+histogram:
+  - ["AMOADD_[WD].*", 1.0]
+  - ["AMOAND_[WD].*", 1.0]
+  - ["AMOMAXU_[WD].*", 1.0]
+  - ["AMOMAX_[WD].*", 1.0]
+  - ["AMOMINU_[WD].*", 1.0]
+  - ["AMOMIN_[WD].*", 1.0]
+  - ["AMOOR_[WD].*", 1.0]
+  - ["AMOSWAP_[WD].*", 1.0]
+  - ["AMOXOR_[WD].*", 1.0]
+  - ["ADD", 1.0]
+  - ["ADDI", 1.0]
+  - ["AND", 1.0]
+  - ["ANDI", 1.0]
+  - ["AUIPC", 1.0]
+  - ["BEQ", 3.0]
+  - ["BGE", 3.0]
+  - ["BGEU", 3.0]
+  - ["BLT", 3.0]
+  - ["BLTU", 3.0]
+  - ["BNE", 3.0]
+  - ["L[BHWD][U]?", 1.0]
+  - ["S[BHWD]", 1.0]
+  - ["LUI", 1.0]
+  - ["OR", 1.0]
+  - ["ORI", 1.0]
+  - ["SLL", 1.0]
+  - ["SLLI", 1.0]
+  - ["SLT", 1.0]
+  - ["SLTI", 1.0]
+  - ["SLTIU", 1.0]
+  - ["SLTU", 1.0]
+  - ["SRA", 1.0]
+  - ["SRAI", 1.0]
+  - ["SRL", 1.0]
+  - ["SRLI", 1.0]
+  - ["SUB", 1.0]
+  - ["XOR", 1.0]
+  - ["XORI", 1.0]
--- a/ci-tests/snippy-tests/basic.yaml
+++ b/ci-tests/snippy-tests/basic.yaml
@ -0,0 +1,57 @@
+options:
+  num-instrs: 1000
+  entry-point: SNIPPY_ENTRY
+  model-plugin: None
+
+sections:
+  - name:        0
+    VMA:       0x80000000
+    SIZE:      0x10000
+    LMA:       0x80000000
+    ACCESS:    r
+  - name:        1
+    VMA:       0x80020000
+    SIZE:      0x20000
+    LMA:       0x80020000
+    ACCESS:    rx
+  - name:        2
+    VMA:       0x80040000
+    SIZE:      0x10000
+    LMA:       0x80040000
+    ACCESS:    rw
+  - name:      stack
+    VMA:       0x80050000
+    SIZE:      0x10000
+    LMA:       0x80050000
+    ACCESS:    rw
+
+histogram:
+  - ["ADD", 1.0]
+  - ["ADDI", 1.0]
+  - ["AND", 1.0]
+  - ["ANDI", 1.0]
+  - ["AUIPC", 1.0]
+  - ["BEQ", 3.0]
+  - ["BGE", 3.0]
+  - ["BGEU", 3.0]
+  - ["BLT", 3.0]
+  - ["BLTU", 3.0]
+  - ["BNE", 3.0]
+  - ["L[BHWD][U]?", 1.0]
+  - ["S[BHWD]", 1.0]
+  - ["LUI", 1.0]
+  - ["OR", 1.0]
+  - ["ORI", 1.0]
+  - ["SLL", 1.0]
+  - ["SLLI", 1.0]
+  - ["SLT", 1.0]
+  - ["SLTI", 1.0]
+  - ["SLTIU", 1.0]
+  - ["SLTU", 1.0]
+  - ["SRA", 1.0]
+  - ["SRAI", 1.0]
+  - ["SRL", 1.0]
+  - ["SRLI", 1.0]
+  - ["SUB", 1.0]
+  - ["XOR", 1.0]
+  - ["XORI", 1.0]
--- a/ci-tests/snippy-tests/boot-code-f.s
+++ b/ci-tests/snippy-tests/boot-code-f.s
@ -0,0 +1,41 @@
+.option norvc
+
+.global _entry
+.global fromhost
+.global tohost
+
+.text
+_entry:
+  la t0, exception_handler
+  csrw mtvec, t0
+  csrr t1, mstatus
+# Setting bit number 13 (mstatus.FS)
+  li t3, 1
+  slli t3, t3, 13
+  or t1, t1, t3
+  csrw mstatus, t1
+  la t0, SNIPPY_ENTRY
+  jalr t0
+
+exception_handler:
+  csrr x10, mcause
+# In case of breakpoint (Interrupt = 0, Exception code = 3) we finalize.
+# Otherwise it's not the expected behavior and we go into an infinite loop.
+  li x11, 3
+  beq x10, x11, exit
+  j infinite_loop
+
+exit:
+  li ra, 1
+  la sp, tohost
+  sw ra, 0(sp)
+
+infinite_loop:
+  j infinite_loop
+
+.balign 64
+tohost:
+.8byte 0x0
+.balign 64
+fromhost:
+.8byte 0x0
--- a/ci-tests/snippy-tests/boot-code.s
+++ b/ci-tests/snippy-tests/boot-code.s
@ -0,0 +1,30 @@
+.option norvc
+.global _entry
+.global fromhost
+.global tohost
+.text
+_entry:
+  la t0, exception_handler
+  csrw mtvec, t0
+  la t0, SNIPPY_ENTRY
+  jalr t0
+  j exit
+exception_handler:
+  csrr x10, mcause
+# In case of breakpoint (Interrupt = 0, Exception code = 3) we finalize.
+# Otherwise it's not the expected behavior and we go into an infinite loop.
+  li x11, 3
+  beq x10, x11, exit
+  j infinite_loop
+exit:
+  li ra, 1
+  la sp, tohost
+  sw ra, 0(sp)
+infinite_loop:
+  j infinite_loop
+.balign 64
+tohost:
+.8byte 0x0
+fromhost:
+.8byte 0x0
+
--- a/ci-tests/snippy-tests/compressed.yaml
+++ b/ci-tests/snippy-tests/compressed.yaml
@ -0,0 +1,76 @@
+options:
+  mattr: +c
+  num-instrs: 1000
+  entry-point: SNIPPY_ENTRY
+  model-plugin: None
+
+sections:
+  - name:        0
+    VMA:       0x80000000
+    SIZE:      0x10000
+    LMA:       0x80000000
+    ACCESS:    r
+  - name:        1
+    VMA:       0x80020000
+    SIZE:      0x20000
+    LMA:       0x80020000
+    ACCESS:    rx
+  - name:        2
+    VMA:       0x80040000
+    SIZE:      0x10000
+    LMA:       0x80040000
+    ACCESS:    rw
+  - name:      stack
+    VMA:       0x80050000
+    SIZE:      0x10000
+    LMA:       0x80050000
+    ACCESS:    rw
+
+histogram:
+  - ["C_ADD.*", 1.0]
+  - ["C_AND", 1.0]
+  - ["C_ANDI", 1.0]
+  - ["C_BEQZ", 5.0]
+  - ["C_BNEZ", 5.0]
+  - ["C_L[WD]", 1.0]
+  - ["C_S[WD]", 1.0]
+  - ["C_LI", 1.0]
+  - ["C_LUI", 1.0]
+  - ["C_MV", 1.0]
+  - ["C_OR", 1.0]
+  - ["C_SLLI", 1.0]
+  - ["C_SRAI", 1.0]
+  - ["C_SRLI", 1.0]
+  - ["C_SUB[W]?", 1.0]
+  - ["C_XOR", 1.0]
+  - ["ADD", 1.0]
+  - ["ADDI", 1.0]
+  - ["AND", 1.0]
+  - ["ANDI", 1.0]
+  - ["AUIPC", 1.0]
+  - ["BEQ", 3.0]
+  - ["BGE", 3.0]
+  - ["BGEU", 3.0]
+  - ["BLT", 3.0]
+  - ["BLTU", 3.0]
+  - ["BNE", 3.0]
+  - ["L[BHWD][U]?", 1.0]
+  - ["S[BHWD]", 1.0]
+  - ["LUI", 1.0]
+  - ["OR", 1.0]
+  - ["ORI", 1.0]
+  - ["SB", 1.0]
+  - ["SH", 1.0]
+  - ["SLL", 1.0]
+  - ["SLLI", 1.0]
+  - ["SLT", 1.0]
+  - ["SLTI", 1.0]
+  - ["SLTIU", 1.0]
+  - ["SLTU", 1.0]
+  - ["SRA", 1.0]
+  - ["SRAI", 1.0]
+  - ["SRL", 1.0]
+  - ["SRLI", 1.0]
+  - ["SUB", 1.0]
+  - ["XOR", 1.0]
+  - ["XORI", 1.0]
--- a/ci-tests/snippy-tests/double-fp.yaml
+++ b/ci-tests/snippy-tests/double-fp.yaml
@ -0,0 +1,109 @@
+options:
+  mattr: +f,+d
+  num-instrs: 1000
+  entry-point: SNIPPY_ENTRY
+  model-plugin: None
+
+sections:
+  - name:        0
+    VMA:       0x80000000
+    SIZE:      0x10000
+    LMA:       0x80000000
+    ACCESS:    r
+  - name:        1
+    VMA:       0x80020000
+    SIZE:      0x20000
+    LMA:       0x80020000
+    ACCESS:    rx
+  - name:        2
+    VMA:       0x80040000
+    SIZE:      0x10000
+    LMA:       0x80040000
+    ACCESS:    rw
+  - name:      stack
+    VMA:       0x80050000
+    SIZE:      0x10000
+    LMA:       0x80050000
+    ACCESS:    rw
+
+histogram:
+  - ["FMADD_S", 1.0]
+  - ["FMSUB_S", 1.0]
+  - ["FNMSUB_S", 1.0]
+  - ["FNMADD_S", 1.0]
+  - ["FADD_S", 1.0]
+  - ["FSUB_S", 1.0]
+  - ["FMUL_S", 1.0]
+  - ["FDIV_S", 1.0]
+  - ["FSQRT_S", 1.0]
+  - ["FSGNJ_S", 1.0]
+  - ["FSGNJN_S", 1.0]
+  - ["FSGNJX_S", 1.0]
+  - ["FMIN_S", 1.0]
+  - ["FMAX_S", 1.0]
+  - ["FCVT_[WL]_S", 1.0]
+  - ["FCVT_[WL]U_S", 1.0]
+  - ["FEQ_S", 1.0]
+  - ["FLT_S", 1.0]
+  - ["FLE_S", 1.0]
+  - ["FCLASS_S", 1.0]
+  - ["FCVT_S_[WL]", 1.0]
+  - ["FCVT_S_[WL]U", 1.0]
+  - ["FMV_.*", 1.0]
+  - ["FL[WD]", 1.0]
+  - ["FS[WD]", 1.0]
+  - ["FMADD_D", 1.0]
+  - ["FMSUB_D", 1.0]
+  - ["FNMSUB_D", 1.0]
+  - ["FNMADD_D", 1.0]
+  - ["FADD_D", 1.0]
+  - ["FSUB_D", 1.0]
+  - ["FMUL_D", 1.0]
+  - ["FDIV_D", 1.0]
+  - ["FSQRT_D", 1.0]
+  - ["FSGNJ_D", 1.0]
+  - ["FSGNJN_D", 1.0]
+  - ["FSGNJX_D", 1.0]
+  - ["FMIN_D", 1.0]
+  - ["FMAX_D", 1.0]
+  - ["FCVT_[WL]_D", 1.0]
+  - ["FCVT_[WL]U_D", 1.0]
+  - ["FCVT_D_[WL]", 1.0]
+  - ["FCVT_D_[WL]U", 1.0]
+  - ["FEQ_D", 1.0]
+  - ["FLT_D", 1.0]
+  - ["FLE_D", 1.0]
+  - ["FCLASS_D", 1.0]
+  - ["FCVT_S_D", 1.0]
+  - ["FCVT_D_S", 1.0]
+  - ["FL[WD]", 1.0]
+  - ["FS[WD]", 1.0]
+  - ["ADD", 1.0]
+  - ["ADDI", 1.0]
+  - ["AND", 1.0]
+  - ["ANDI", 1.0]
+  - ["AUIPC", 1.0]
+  - ["BEQ", 3.0]
+  - ["BGE", 3.0]
+  - ["BGEU", 3.0]
+  - ["BLT", 3.0]
+  - ["BLTU", 3.0]
+  - ["BNE", 3.0]
+  - ["L[BHWD][U]?", 1.0]
+  - ["S[BHWD]", 1.0]
+  - ["LUI", 1.0]
+  - ["OR", 1.0]
+  - ["ORI", 1.0]
+  - ["SLL", 1.0]
+  - ["SLLI", 1.0]
+  - ["SLT", 1.0]
+  - ["SLTI", 1.0]
+  - ["SLTIU", 1.0]
+  - ["SLTU", 1.0]
+  - ["SRA", 1.0]
+  - ["SRAI", 1.0]
+  - ["SRL", 1.0]
+  - ["SRLI", 1.0]
+  - ["SUB", 1.0]
+  - ["XOR", 1.0]
+  - ["XORI", 1.0]
--- a/ci-tests/snippy-tests/linker-entry.ld
+++ b/ci-tests/snippy-tests/linker-entry.ld
@ -0,0 +1 @@
+ENTRY(_entry)
--- a/ci-tests/snippy-tests/single-fp.yaml
+++ b/ci-tests/snippy-tests/single-fp.yaml
@ -0,0 +1,84 @@
+options:
+  mattr: +f
+  num-instrs: 1000
+  entry-point: SNIPPY_ENTRY
+  model-plugin: None
+
+sections:
+    - name:        0
+      VMA:       0x80000000
+      SIZE:      0x10000
+      LMA:       0x80000000
+      ACCESS:    r
+    - name:        1
+      VMA:       0x80020000
+      SIZE:      0x20000
+      LMA:       0x80020000
+      ACCESS:    rx
+    - name:        2
+      VMA:       0x80040000
+      SIZE:      0x10000
+      LMA:       0x80040000
+      ACCESS:    rw
+    - name:      stack
+      VMA:       0x80050000
+      SIZE:      0x10000
+      LMA:       0x80050000
+      ACCESS:    rw
+
+histogram:
+  - ["FMADD_S", 1.0]
+  - ["FMSUB_S", 1.0]
+  - ["FNMSUB_S", 1.0]
+  - ["FNMADD_S", 1.0]
+  - ["FADD_S", 1.0]
+  - ["FSUB_S", 1.0]
+  - ["FMUL_S", 1.0]
+  - ["FDIV_S", 1.0]
+  - ["FSQRT_S", 1.0]
+  - ["FSGNJ_S", 1.0]
+  - ["FSGNJN_S", 1.0]
+  - ["FSGNJX_S", 1.0]
+  - ["FMIN_S", 1.0]
+  - ["FMAX_S", 1.0]
+  - ["FMV_X_[WL]", 1.0]
+  - ["FEQ_S", 1.0]
+  - ["FLT_S", 1.0]
+  - ["FLE_S", 1.0]
+  - ["FCLASS_S", 1.0]
+  - ["FCVT_S_[WL]", 1.0]
+  - ["FCVT_S_[WL]U", 1.0]
+  - ["FCVT_[WL]U_S", 1.0]
+  - ["FCVT_[WL]_S", 1.0]
+  - ["FMV_[WL]_X", 1.0]
+  - ["FL[WD]", 1.0]
+  - ["FS[WD]", 1.0]
+  - ["ADD", 1.0]
+  - ["ADDI", 1.0]
+  - ["AND", 1.0]
+  - ["ANDI", 1.0]
+  - ["AUIPC", 1.0]
+  - ["BEQ", 3.0]
+  - ["BGE", 3.0]
+  - ["BGEU", 3.0]
+  - ["BLT", 3.0]
+  - ["BLTU", 3.0]
+  - ["BNE", 3.0]
+  - ["L[BHWD][U]?", 1.0]
+  - ["S[BHWD]", 1.0]
+  - ["LUI", 1.0]
+  - ["OR", 1.0]
+  - ["ORI", 1.0]
+  - ["SLL", 1.0]
+  - ["SLLI", 1.0]
+  - ["SLT", 1.0]
+  - ["SLTI", 1.0]
+  - ["SLTIU", 1.0]
+  - ["SLTU", 1.0]
+  - ["SRA", 1.0]
+  - ["SRAI", 1.0]
+  - ["SRL", 1.0]
+  - ["SRLI", 1.0]
+  - ["SUB", 1.0]
+  - ["XOR", 1.0]
+  - ["XORI", 1.0]
--- a/ci-tests/test-spike
+++ b/ci-tests/test-spike
@ -27,6 +27,14 @@ riscv64-linux-gnu-gcc -static -O2 -o dummy-slliuw $CI/dummy-slliuw.c
 riscv64-linux-gnu-gcc -static -O2 -o customcsr $CI/customcsr.c
 riscv64-linux-gnu-gcc -static -O2 -o atomics $CI/atomics.c

+# run snippy-based tests
+wget https://github.com/syntacore/snippy/releases/download/snippy-2.0/snippy-2.0.tar.gz
+tar xzf "snippy-2.0.tar.gz"
+# test that snippy runs
+./llvm-snippy --version | grep "Snippy version"
+"$ROOT"/ci-tests/generate-snippy-tests.sh "$RUN" "$ROOT"/ci-tests/snippy-tests
+"$ROOT"/ci-tests/run-snippy-tests.sh snippy-tests "$INSTALL"/bin/spike
+
 # check that including sim.h in an external project works
 g++ -std=c++2a -I$INSTALL/include -L$INSTALL/lib $CI/testlib.cc -lriscv -o test-libriscv
 g++ -std=c++2a -I$INSTALL/include -L$INSTALL/lib $CI/test-customext.cc -lriscv -o test-customext
--- a/spike-ci.tar
+++ b/spike-ci.tar