12 — Capstone: word-frequency report

A small end-to-end program pulling the course together: read text, tokenise it, count words in a map, and return the top-N by frequency. The example is live — edit the input and press Run.

The pipeline

Tokenize lower-cases and splits on non-letter/digit boundaries (strings.FieldsFunc). Count reads an io.Reader and tallies into a map. TopN copies the map into a slice and sorts it with slices.SortFunc — count descending, then word ascending for ties. Report wires them together. Each piece draws on an earlier module.

package main

import (
    "fmt"
    "io"
    "slices"
    "strings"
    "unicode"
)

func Tokenize(s string) []string {
    s = strings.ToLower(s)
    return strings.FieldsFunc(s, func(r rune) bool {
        return !unicode.IsLetter(r) && !unicode.IsDigit(r)
    })
}

func Count(r io.Reader) (map[string]int, error) {
    data, err := io.ReadAll(r)
    if err != nil {
        return nil, err
    }
    counts := map[string]int{}
    for _, w := range Tokenize(string(data)) {
        counts[w]++
    }
    return counts, nil
}

type WordFreq struct {
    Word  string
    Count int
}

func TopN(counts map[string]int, n int) []WordFreq {
    if n <= 0 {
        return []WordFreq{}
    }
    freqs := make([]WordFreq, 0, len(counts))
    for w, c := range counts {
        freqs = append(freqs, WordFreq{w, c})
    }
    slices.SortFunc(freqs, func(a, b WordFreq) int {
        if a.Count != b.Count {
            return b.Count - a.Count          // count descending
        }
        return strings.Compare(a.Word, b.Word) // ties: alphabetical
    })
    if n > len(freqs) {
        n = len(freqs)
    }
    return freqs[:n]
}

func Report(r io.Reader, n int) ([]WordFreq, error) {
    counts, err := Count(r)
    if err != nil {
        return nil, err
    }
    return TopN(counts, n), nil
}

func main() {
    text := "The quick brown fox. The quick fox! THE dog."
    rep, _ := Report(strings.NewReader(text), 2)
    for _, wf := range rep {
        fmt.Printf("%s %d\n", wf.Word, wf.Count)
    }
}

Output:

the 3
fox 2

Counts are the:3, quick:2, fox:2, brown:1, dog:1. The top 2 by count are the (3) and then — tie at 2 — fox before quick alphabetically. Try widening to Report(..., 5) to see the full ordering.

What each piece draws on

Next: 13 — Testing.