clerk @ bcc8624

···
        
        1
        +#!/usr/bin/env python3

      
        
        2
        +"""Fail if any benchmark regressed beyond the threshold.

      
        
        3
        +

      
        
        4
        +Reads benchstat CSV comparison (old vs new) from stdin. benchstat only

      
        
        5
        +reports a numeric delta for statistically significant changes (p < 0.05);

      
        
        6
        +noisy comparisons are marked "~" and ignored.

      
        
        7
        +"""

      
        
        8
        +import csv

      
        
        9
        +import sys

      
        
        10
        +

      
        
        11
        +THRESHOLD = 20.0  # percent slower

      
        
        12
        +

      
        
        13
        +def delta_pct(s):

      
        
        14
        +    s = s.strip()

      
        
        15
        +    if not s or s == "~":

      
        
        16
        +        return None

      
        
        17
        +    sign = 1.0

      
        
        18
        +    if s.startswith("+"):

      
        
        19
        +        s = s[1:]

      
        
        20
        +    elif s.startswith("-"):

      
        
        21
        +        sign = -1.0

      
        
        22
        +        s = s[1:]

      
        
        23
        +    s = s.rstrip("%")

      
        
        24
        +    if s in ("∞", "Inf", "inf", "+Inf"):

      
        
        25
        +        return sign * float("inf")

      
        
        26
        +    try:

      
        
        27
        +        return sign * float(s)

      
        
        28
        +    except ValueError:

      
        
        29
        +        return None

      
        
        30
        +

      
        
        31
        +def main():

      
        
        32
        +    regressions = []

      
        
        33
        +    for row in csv.reader(sys.stdin):

      
        
        34
        +        if len(row) < 7 or row[0] in ("", "geomean"):

      
        
        35
        +            continue

      
        
        36
        +        d = delta_pct(row[5])  # vs base column

      
        
        37
        +        if d is not None and d > THRESHOLD:

      
        
        38
        +            regressions.append((row[0], d))

      
        
        39
        +

      
        
        40
        +    if not regressions:

      
        
        41
        +        print("no regressions beyond +%.0f%%" % THRESHOLD)

      
        
        42
        +        return 0

      
        
        43
        +

      
        
        44
        +    print("performance regressions beyond +%.0f%%:" % THRESHOLD)

      
        
        45
        +    for name, d in sorted(regressions):

      
        
        46
        +        if d == float("inf"):

      
        
        47
        +            print("  %s: +∞ (new cost where none before)" % name)

      
        
        48
        +        else:

      
        
        49
        +            print("  %s: +%.1f%%" % (name, d))

      
        
        50
        +    return 1

      
        
        51
        +

      
        
        52
        +

      
        
        53
        +if __name__ == "__main__":

      
        
        54
        +    sys.exit(main())

      

···
        
        1
        +name: benchmarks

      
        
        2
        +

      
        
        3
        +on:

      
        
        4
        +  push:

      
        
        5
        +    branches: [main]

      
        
        6
        +  pull_request:

      
        
        7
        +  workflow_dispatch:

      
        
        8
        +

      
        
        9
        +concurrency:

      
        
        10
        +  group: benchmarks-${{ github.ref }}

      
        
        11
        +  cancel-in-progress: true

      
        
        12
        +

      
        
        13
        +permissions:

      
        
        14
        +  contents: read

      
        
        15
        +

      
        
        16
        +jobs:

      
        
        17
        +  bench:

      
        
        18
        +    runs-on: ubuntu-latest

      
        
        19
        +    timeout-minutes: 30

      
        
        20
        +    steps:

      
        
        21
        +      - uses: actions/checkout@v4

      
        
        22
        +      - uses: actions/setup-go@v5

      
        
        23
        +        with:

      
        
        24
        +          go-version-file: go.mod

      
        
        25
        +

      
        
        26
        +      - name: Install benchstat

      
        
        27
        +        run: |

      
        
        28
        +          go install golang.org/x/perf/cmd/benchstat@latest

      
        
        29
        +          echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"

      
        
        30
        +

      
        
        31
        +      - name: Run benchmarks

      
        
        32
        +        run: go test -bench . -benchmem -run '^$' -count=10 -benchtime=200x ./... > bench.txt 2>&1

      
        
        33
        +

      
        
        34
        +      - name: Restore baseline

      
        
        35
        +        id: baseline

      
        
        36
        +        uses: actions/cache/restore@v4

      
        
        37
        +        with:

      
        
        38
        +          path: bench-baseline.txt

      
        
        39
        +          key: bench-${{ runner.os }}-${{ github.sha }}

      
        
        40
        +          restore-keys: bench-${{ runner.os }}-

      
        
        41
        +

      
        
        42
        +      - name: Print comparison

      
        
        43
        +        if: steps.baseline.outputs.cache-hit == 'true'

      
        
        44
        +        run: benchstat bench-baseline.txt bench.txt

      
        
        45
        +

      
        
        46
        +      - name: Check for regressions

      
        
        47
        +        if: steps.baseline.outputs.cache-hit == 'true'

      
        
        48
        +        run: |

      
        
        49
        +          benchstat -format csv bench-baseline.txt bench.txt 2>/dev/null | \

      
        
        50
        +            python3 .github/scripts/check-bench-regression.py

      
        
        51
        +

      
        
        52
        +      - name: Save baseline

      
        
        53
        +        if: github.ref == 'refs/heads/main'

      
        
        54
        +        uses: actions/cache/save@v4

      
        
        55
        +        with:

      
        
        56
        +          path: bench.txt

      
        
        57
        +          key: bench-${{ runner.os }}-${{ github.sha }}

      

···
        
        1
        +name: CI

      
        
        2
        +

      
        
        3
        +on:

      
        
        4
        +  push:

      
        
        5
        +  pull_request:

      
        
        6
        +

      
        
        7
        +jobs:

      
        
        8
        +  test:

      
        
        9
        +    runs-on: ubuntu-latest

      
        
        10
        +    timeout-minutes: 10

      
        
        11
        +    steps:

      
        
        12
        +      - uses: actions/checkout@v4

      
        
        13
        +      - uses: actions/setup-go@v5

      
        
        14
        +        with:

      
        
        15
        +          go-version-file: go.mod

      
        
        16
        +

      
        
        17
        +      - run: go build ./...

      
        
        18
        +      - run: go vet ./...

      
        
        19
        +

      
        
        20
        +      - name: Test with coverage

      
        
        21
        +        run: go test -coverprofile=coverage.out ./...

      
        
        22
        +

      
        
        23
        +      - name: Coverage summary

      
        
        24
        +        run: go tool cover -func=coverage.out | tail -1

      
        
        25
        +

      
        
        26
        +      - name: Generate HTML coverage report

      
        
        27
        +        run: go tool cover -html=coverage.out -o coverage.html

      
        
        28
        +

      
        
        29
        +      - uses: actions/upload-artifact@v4

      
        
        30
        +        with:

      
        
        31
        +          name: coverage-report

      
        
        32
        +          path: |

      
        
        33
        +            coverage.html

      
        
        34
        +            coverage.out

      
        
        35
        +

      
        
        36
        +  fuzz:

      
        
        37
        +    runs-on: ubuntu-latest

      
        
        38
        +    timeout-minutes: 10

      
        
        39
        +    steps:

      
        
        40
        +      - uses: actions/checkout@v4

      
        
        41
        +      - uses: actions/setup-go@v5

      
        
        42
        +        with:

      
        
        43
        +          go-version-file: go.mod

      
        
        44
        +

      
        
        45
        +      - name: Fuzz lexer

      
        
        46
        +        run: go test -fuzz=FuzzLexer -fuzztime=90s ./journal/lexer/

      
        
        47
        +

      
        
        48
        +      - name: Fuzz parser

      
        
        49
        +        run: go test -fuzz=FuzzParser -fuzztime=90s ./journal/parser/