File I/O in Go

Go’s file APIs live mainly in os (open, read, write, stat), bufio (buffered reading and writing), io (generic streams) and path/filepath. Every program below creates the files it needs in the current directory, so you can run them as they are.

Whole files: os.WriteFile and os.ReadFile

For small files, read or write everything in one call.

package main

import (
	"errors"
	"fmt"
	"io/fs"
	"os"
)

func main() {
	err := os.WriteFile("notes.txt", []byte("first line\nsecond line\n"), 0o644)
	if err != nil {
		panic(err)
	}

	data, err := os.ReadFile("notes.txt")
	fmt.Printf("%d bytes, err=%v\n%s", len(data), err, data)

	info, _ := os.Stat("notes.txt")
	fmt.Println(info.Name(), info.Size(), info.Mode())

	_, err = os.ReadFile("missing.txt")
	fmt.Println(err)
	fmt.Println(errors.Is(err, fs.ErrNotExist))
}

$ go run whole.go
23 bytes, err=<nil>
first line
second line
notes.txt 23 -rw-r--r--
open missing.txt: no such file or directory
true

os.WriteFile creates or truncates the file; 0o644 is the permission for a newly created file (owner read/write, everyone else read), which Mode() prints as -rw-r--r-- — the process umask can remove bits from it. A missing file is an ordinary error value; test for it with errors.Is(err, fs.ErrNotExist) rather than comparing message strings.

Line by line with bufio.Scanner

bufio.Scanner reads a file a line at a time, so memory use doesn’t grow with the file size.

package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"
)

func main() {
	os.WriteFile("log.txt", []byte("INFO start\nERROR disk full\nINFO retry\nERROR timeout\n"), 0o644)

	f, err := os.Open("log.txt")
	if err != nil {
		panic(err)
	}
	defer f.Close()

	sc := bufio.NewScanner(f)
	n := 0
	for sc.Scan() {
		n++
		if line := sc.Text(); strings.HasPrefix(line, "ERROR") {
			fmt.Println(n, line)
		}
	}
	if err := sc.Err(); err != nil {
		fmt.Println("scan error:", err)
	}
}

$ go run scan.go
2 ERROR disk full
4 ERROR timeout

Scan returns false both at the end of the file and on an error, so check sc.Err() after the loop. Text() strips the newline. defer f.Close() right after a successful Open is the standard way to make sure the file is closed on every return path.

That sc.Err() check is not a formality. A Scanner refuses lines longer than 64 KB by default:

package main

import (
	"bufio"
	"fmt"
	"strings"
)

func main() {
	input := "short\n" + strings.Repeat("x", 70_000) + "\nafter\n"

	sc := bufio.NewScanner(strings.NewReader(input))
	for sc.Scan() {
		fmt.Println("line of", len(sc.Text()), "bytes")
	}
	fmt.Println("err:", sc.Err())

	sc = bufio.NewScanner(strings.NewReader(input))
	sc.Buffer(make([]byte, 0, 64*1024), 1<<20) // allow lines up to 1 MB
	for sc.Scan() {
		fmt.Println("line of", len(sc.Text()), "bytes")
	}
	fmt.Println("err:", sc.Err())
}

$ go run toolong.go
line of 5 bytes
err: bufio.Scanner: token too long
line of 5 bytes
line of 70000 bytes
line of 5 bytes
err: <nil>

The first loop silently stopped after short, and the line after was never seen; only sc.Err() reveals why. Minified JSON or long log lines hit this in production. sc.Buffer raises the limit; for lines of unknown length, bufio.Reader.ReadString('\n') has no limit at all.

Buffered writing and the missing Flush

bufio.Writer collects small writes in memory and writes them out in large chunks — but only when it is flushed.

package main

import (
	"bufio"
	"fmt"
	"os"
)

func write(name string, flush bool) {
	f, err := os.Create(name)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	w := bufio.NewWriter(f)
	for i := range 100 {
		fmt.Fprintf(w, "row %d\n", i)
	}
	if flush {
		if err := w.Flush(); err != nil {
			panic(err)
		}
	}
}

func main() {
	write("noflush.txt", false)
	write("flushed.txt", true)
	for _, name := range []string{"noflush.txt", "flushed.txt"} {
		info, _ := os.Stat(name)
		fmt.Println(name, info.Size(), "bytes")
	}
}

$ go run flush.go
noflush.txt 0 bytes
flushed.txt 690 bytes

Both functions wrote the same 100 rows, but without Flush they stayed in the 4 KB buffer and were discarded when the function returned; the file exists and is empty. No error was reported anywhere. Always Flush a bufio.Writer and check the error — a full disk shows up there.

Appending and checking Close

os.OpenFile takes flags. O_APPEND|O_CREATE|O_WRONLY creates the file if needed and adds to the end.

package main

import (
	"fmt"
	"os"
)

func appendLine(name, line string) (err error) {
	f, err := os.OpenFile(name, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
	if err != nil {
		return err
	}
	defer func() {
		if cerr := f.Close(); cerr != nil && err == nil {
			err = cerr // a failed Close can mean lost data
		}
	}()
	_, err = fmt.Fprintln(f, line)
	return err
}

func main() {
	os.Remove("audit.log")
	for _, ev := range []string{"login ann", "export report", "logout ann"} {
		if err := appendLine("audit.log", ev); err != nil {
			fmt.Println("error:", err)
		}
	}
	data, _ := os.ReadFile("audit.log")
	fmt.Print(string(data))
}

$ go run appendlog.go
login ann
export report
logout ann

Each call opens, appends and closes, so earlier lines survive. For files you write, the error from Close matters: the operating system may only report a failed write when the file is closed. The deferred closure turns a Close error into the function’s error unless an earlier error is already being returned.

Paths and directories

path/filepath builds and splits paths with the operating system’s separator; filepath.WalkDir visits a tree.

package main

import (
	"fmt"
	"io/fs"
	"os"
	"path/filepath"
)

func main() {
	p := filepath.Join("data", "2024", "report.csv")
	fmt.Println(p, filepath.Dir(p), filepath.Base(p), filepath.Ext(p))

	os.RemoveAll("data")
	os.MkdirAll(filepath.Join("data", "2024"), 0o755)
	os.MkdirAll(filepath.Join("data", "2025"), 0o755)
	for _, f := range []string{"2024/report.csv", "2024/notes.txt", "2025/report.csv"} {
		os.WriteFile(filepath.Join("data", f), []byte("x"), 0o644)
	}

	filepath.WalkDir("data", func(path string, d fs.DirEntry, err error) error {
		if err != nil {
			return err
		}
		if !d.IsDir() && filepath.Ext(path) == ".csv" {
			fmt.Println("csv:", path)
		}
		return nil
	})
}

$ go run walk.go
data/2024/report.csv data/2024 report.csv .csv
csv: data/2024/report.csv
csv: data/2025/report.csv

filepath.Join inserts separators and cleans the result, so you never concatenate "/" by hand (on Windows it produces backslashes). WalkDir visits entries in lexical order, which makes its output stable. Returning filepath.SkipDir from the callback skips a directory’s contents.

JSON files

encoding/json reads and writes structs; json.Encoder and json.Decoder work directly on files and other streams.

package main

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"
)

type Config struct {
	Host  string `json:"host"`
	Port  int    `json:"port"`
	Debug bool   `json:"debug,omitempty"`
}

func main() {
	f, _ := os.Create("config.json")
	enc := json.NewEncoder(f)
	enc.SetIndent("", "  ")
	enc.Encode(Config{Host: "localhost", Port: 8080})
	f.Close()

	data, _ := os.ReadFile("config.json")
	fmt.Print(string(data))

	var cfg Config
	err := json.Unmarshal(data, &cfg)
	fmt.Printf("%+v %v\n", cfg, err)

	dec := json.NewDecoder(strings.NewReader(`{"host":"x","port":"80"}`))
	dec.DisallowUnknownFields()
	fmt.Println(dec.Decode(&cfg))
}

$ go run jsonfile.go
{
  "host": "localhost",
  "port": 8080
}
{Host:localhost Port:8080 Debug:false} <nil>
json: cannot unmarshal string into Go struct field Config.port of type int

omitempty left debug out because it was false. The last decode fails with a precise message: port is a string in the input but an int in the struct. DisallowUnknownFields additionally rejects keys the struct doesn’t have, which catches typos in config files that Unmarshal would silently ignore.

Practice

1. Count the lines, words and bytes of log.txt from scan.go, like wc.

package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"
)

func main() {
	os.WriteFile("log.txt", []byte("INFO start\nERROR disk full\nINFO retry\nERROR timeout\n"), 0o644)
	f, err := os.Open("log.txt")
	if err != nil {
		panic(err)
	}
	defer f.Close()

	lines, words, bytes := 0, 0, 0
	sc := bufio.NewScanner(f)
	for sc.Scan() {
		lines++
		words += len(strings.Fields(sc.Text()))
		bytes += len(sc.Bytes()) + 1 // +1 for the newline Scan removed
	}
	fmt.Println(lines, words, bytes)
}

$ go run p1.go
4 9 52
$ wc log.txt
 4  9 52 log.txt

2. Copy notes.txt to notes.bak with io.Copy, and print how many bytes were copied.

package main

import (
	"fmt"
	"io"
	"os"
)

func copyFile(dst, src string) (int64, error) {
	in, err := os.Open(src)
	if err != nil {
		return 0, err
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return 0, err
	}
	n, err := io.Copy(out, in)
	if cerr := out.Close(); err == nil {
		err = cerr
	}
	return n, err
}

func main() {
	os.WriteFile("notes.txt", []byte("first line\nsecond line\n"), 0o644)
	fmt.Println(copyFile("notes.bak", "notes.txt"))
	fmt.Println(copyFile("x.bak", "nope.txt"))
}

$ go run p2.go
23 <nil>
0 open nope.txt: no such file or directory

3. Use os.ReadDir to list the entries of data/2024 from walk.go with a / after directories.

package main

import (
	"fmt"
	"os"
)

func main() {
	os.MkdirAll("data/2024/archive", 0o755)
	os.WriteFile("data/2024/report.csv", nil, 0o644)

	entries, err := os.ReadDir("data/2024")
	if err != nil {
		panic(err)
	}
	for _, e := range entries {
		name := e.Name()
		if e.IsDir() {
			name += "/"
		}
		fmt.Println(name)
	}
}

$ go run p3.go
archive/
report.csv

4. Save a []Config to servers.json and load it back, printing how many servers were read.

package main

import (
	"encoding/json"
	"fmt"
	"os"
)

type Config struct {
	Host string `json:"host"`
	Port int    `json:"port"`
}

func main() {
	out, _ := json.Marshal([]Config{{"a.local", 80}, {"b.local", 443}})
	if err := os.WriteFile("servers.json", out, 0o644); err != nil {
		panic(err)
	}

	data, _ := os.ReadFile("servers.json")
	var servers []Config
	if err := json.Unmarshal(data, &servers); err != nil {
		panic(err)
	}
	fmt.Println(len(servers), servers[1].Host, string(data))
}

$ go run p4.go
2 b.local [{"host":"a.local","port":80},{"host":"b.local","port":443}]