Skip to content

Commit

Permalink
Support for multiple configuration files (#534)
Browse files Browse the repository at this point in the history
  • Loading branch information
FileGo authored May 25, 2022
1 parent 8d0bd8e commit 0731ebe
Show file tree
Hide file tree
Showing 7 changed files with 156 additions and 36 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 65,7 @@ Blocky is a DNS proxy and ad-blocker for the local network written in Go with fo
* Various REST API endpoints
* CLI tool

- **Simple configuration** - single configuration file in YAML format
- **Simple configuration** - single or multiple configuration files in YAML format

* Simple to maintain
* Simple to backup
Expand Down
2 changes: 1 addition & 1 deletion cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 40,7 @@ Complete documentation is available at https://github.com/0xERR0R/blocky`,
SilenceUsage: true,
}

c.PersistentFlags().StringVarP(&configPath, "config", "c", defaultConfigPath, "path to config file")
c.PersistentFlags().StringVarP(&configPath, "config", "c", defaultConfigPath, "path to config file or folder")
c.PersistentFlags().StringVar(&apiHost, "apiHost", defaultHost, "host of blocky (API). Default overridden by config and CLI.") // nolint:lll
c.PersistentFlags().Uint16Var(&apiPort, "apiPort", defaultPort, "port of blocky (API). Default overridden by config and CLI.") // nolint:lll

Expand Down
41 changes: 37 additions & 4 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 7,7 @@ import (
"io/ioutil"
"net"
"os"
"path/filepath"
"regexp"
"sort"
"strconv"
Expand Down Expand Up @@ -539,15 540,14 @@ type FilteringConfig struct {
// nolint:gochecknoglobals
var config = &Config{}

// LoadConfig creates new config from YAML file
// LoadConfig creates new config from YAML file or a directory containing YAML files
func LoadConfig(path string, mandatory bool) (*Config, error) {
cfg := Config{}
if err := defaults.Set(&cfg); err != nil {
return nil, fmt.Errorf("can't apply default values: %w", err)
}

data, err := ioutil.ReadFile(path)

fs, err := os.Stat(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) && !mandatory {
// config file does not exist
Expand All @@ -557,7 557,40 @@ func LoadConfig(path string, mandatory bool) (*Config, error) {
return config, nil
}

return nil, fmt.Errorf("can't read config file: %w", err)
return nil, fmt.Errorf("can't read config file(s): %w", err)
}

var data []byte

if fs.IsDir() { //nolint:nestif
err = filepath.WalkDir(path, func(filePath string, d os.DirEntry, err error) error {
if err != nil {
return err
}

if path == filePath {
return nil
}

fileData, err := os.ReadFile(filePath)
if err != nil {
return err
}

data = append(data, []byte("\n")...)
data = append(data, fileData...)

return nil
})

if err != nil {
return nil, fmt.Errorf("can't read config files: %w", err)
}
} else {
data, err = ioutil.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("can't read config file: %w", err)
}
}

err = unmarshalConfig(data, &cfg)
Expand Down
81 changes: 52 additions & 29 deletions config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,36 24,27 @@ var _ = Describe("Config", func() {
_, err = LoadConfig("config.yml", true)
Expect(err).Should(Succeed())

Expect(config.DNSPorts).Should(Equal(ListenConfig{"55553", ":55554", "[::1]:55555"}))
Expect(config.Upstream.ExternalResolvers["default"]).Should(HaveLen(3))
Expect(config.Upstream.ExternalResolvers["default"][0].Host).Should(Equal("8.8.8.8"))
Expect(config.Upstream.ExternalResolvers["default"][1].Host).Should(Equal("8.8.4.4"))
Expect(config.Upstream.ExternalResolvers["default"][2].Host).Should(Equal("1.1.1.1"))
Expect(config.CustomDNS.Mapping.HostIPs).Should(HaveLen(2))
Expect(config.CustomDNS.Mapping.HostIPs["my.duckdns.org"][0]).Should(Equal(net.ParseIP("192.168.178.3")))
Expect(config.CustomDNS.Mapping.HostIPs["multiple.ips"][0]).Should(Equal(net.ParseIP("192.168.178.3")))
Expect(config.CustomDNS.Mapping.HostIPs["multiple.ips"][1]).Should(Equal(net.ParseIP("192.168.178.4")))
Expect(config.CustomDNS.Mapping.HostIPs["multiple.ips"][2]).Should(Equal(
net.ParseIP("2001:0db8:85a3:08d3:1319:8a2e:0370:7344")))
Expect(config.Conditional.Mapping.Upstreams).Should(HaveLen(2))
Expect(config.Conditional.Mapping.Upstreams["fritz.box"]).Should(HaveLen(1))
Expect(config.Conditional.Mapping.Upstreams["multiple.resolvers"]).Should(HaveLen(2))
Expect(config.ClientLookup.Upstream.Host).Should(Equal("192.168.178.1"))
Expect(config.ClientLookup.SingleNameOrder).Should(Equal([]uint{2, 1}))
Expect(config.Blocking.BlackLists).Should(HaveLen(2))
Expect(config.Blocking.WhiteLists).Should(HaveLen(1))
Expect(config.Blocking.ClientGroupsBlock).Should(HaveLen(2))
Expect(config.Blocking.BlockTTL).Should(Equal(Duration(time.Minute)))
Expect(config.Blocking.RefreshPeriod).Should(Equal(Duration(2 * time.Hour)))
Expect(config.Filtering.QueryTypes).Should(HaveLen(2))

Expect(config.Caching.MaxCachingTime).Should(Equal(Duration(0)))
Expect(config.Caching.MinCachingTime).Should(Equal(Duration(0)))

Expect(config.DoHUserAgent).Should(Equal("testBlocky"))

Expect(GetConfig()).Should(Not(BeNil()))
defaultTestFileConfig()
})
})
When("Test file does not exist", func() {
It("should fail", func() {
_, err := LoadConfig("../testdata/config-does-not-exist.yaml", true)
Expect(err).Should(Not(Succeed()))
})
})
When("Multiple config files are used", func() {
It("should return a valid config struct", func() {
_, err := LoadConfig("../testdata/config/", true)
Expect(err).Should(Succeed())

defaultTestFileConfig()
})
})
When("Config folder does not exist", func() {
It("should fail", func() {
_, err := LoadConfig("../testdata/does-not-exist-config/", true)
Expect(err).Should(Not(Succeed()))
})
})
When("config file is malformed", func() {
Expand Down Expand Up @@ -618,3 609,35 @@ bootstrapDns:
})
})
})

func defaultTestFileConfig() {
Expect(config.DNSPorts).Should(Equal(ListenConfig{"55553", ":55554", "[::1]:55555"}))
Expect(config.Upstream.ExternalResolvers["default"]).Should(HaveLen(3))
Expect(config.Upstream.ExternalResolvers["default"][0].Host).Should(Equal("8.8.8.8"))
Expect(config.Upstream.ExternalResolvers["default"][1].Host).Should(Equal("8.8.4.4"))
Expect(config.Upstream.ExternalResolvers["default"][2].Host).Should(Equal("1.1.1.1"))
Expect(config.CustomDNS.Mapping.HostIPs).Should(HaveLen(2))
Expect(config.CustomDNS.Mapping.HostIPs["my.duckdns.org"][0]).Should(Equal(net.ParseIP("192.168.178.3")))
Expect(config.CustomDNS.Mapping.HostIPs["multiple.ips"][0]).Should(Equal(net.ParseIP("192.168.178.3")))
Expect(config.CustomDNS.Mapping.HostIPs["multiple.ips"][1]).Should(Equal(net.ParseIP("192.168.178.4")))
Expect(config.CustomDNS.Mapping.HostIPs["multiple.ips"][2]).Should(Equal(
net.ParseIP("2001:0db8:85a3:08d3:1319:8a2e:0370:7344")))
Expect(config.Conditional.Mapping.Upstreams).Should(HaveLen(2))
Expect(config.Conditional.Mapping.Upstreams["fritz.box"]).Should(HaveLen(1))
Expect(config.Conditional.Mapping.Upstreams["multiple.resolvers"]).Should(HaveLen(2))
Expect(config.ClientLookup.Upstream.Host).Should(Equal("192.168.178.1"))
Expect(config.ClientLookup.SingleNameOrder).Should(Equal([]uint{2, 1}))
Expect(config.Blocking.BlackLists).Should(HaveLen(2))
Expect(config.Blocking.WhiteLists).Should(HaveLen(1))
Expect(config.Blocking.ClientGroupsBlock).Should(HaveLen(2))
Expect(config.Blocking.BlockTTL).Should(Equal(Duration(time.Minute)))
Expect(config.Blocking.RefreshPeriod).Should(Equal(Duration(2 * time.Hour)))
Expect(config.Filtering.QueryTypes).Should(HaveLen(2))

Expect(config.Caching.MaxCachingTime).Should(Equal(Duration(0)))
Expect(config.Caching.MinCachingTime).Should(Equal(Duration(0)))

Expect(config.DoHUserAgent).Should(Equal("testBlocky"))

Expect(GetConfig()).Should(Not(BeNil()))
}
12 changes: 11 additions & 1 deletion docs/installation.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 8,7 @@ You can choose one of the following installation options:

## Prepare your configuration

Blocky uses one YAML file as configuration. Create new `config.yaml` with your configuration (
Blocky supports single or multiple YAML files as configuration. Create new `config.yaml` with your configuration (
see [Configuration](configuration.md) for more details and all configuration options).

Simple configuration file, which enables only basic features:
Expand Down Expand Up @@ -128,6 128,16 @@ volumes:
device: //NAS_HOSTNAME/blocky
```

#### Multiple configuration files

For complex setups, splitting the configuration between multiple YAML files might be desired. In this case, folder containing YAML files is passed on startup, Blocky will join all the files.

`./blocky --config ./config/`

!!! warning

Blocky simply joins the multiple YAML files. If a directive (e.g. `upstream`) is repeated in multiple files, the configuration will not load and start will fail.

## Other installation types

!!! warning
Expand Down
18 changes: 18 additions & 0 deletions testdata/config/config1.yaml
Original file line number Diff line number Diff line change
@@ -0,0 1,18 @@
upstream:
default:
- tcp udp:8.8.8.8
- tcp udp:8.8.4.4
- 1.1.1.1
customDNS:
mapping:
my.duckdns.org: 192.168.178.3
multiple.ips: 192.168.178.3,192.168.178.4,2001:0db8:85a3:08d3:1319:8a2e:0370:7344
conditional:
mapping:
fritz.box: tcp udp:192.168.178.1
multiple.resolvers: tcp udp:192.168.178.1,tcp udp:192.168.178.2
filtering:
queryTypes:
- AAAA
- A

36 changes: 36 additions & 0 deletions testdata/config/config2.yaml
Original file line number Diff line number Diff line change
@@ -0,0 1,36 @@
blocking:
blackLists:
ads:
- https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt
- https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
- https://mirror1.malwaredomains.com/files/justdomains
- http://sysctl.org/cameleon/hosts
- https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist
- https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt
special:
- https://hosts-file.net/ad_servers.txt
whiteLists:
ads:
- whitelist.txt
clientGroupsBlock:
default:
- ads
- special
Laptop-D.fritz.box:
- ads
blockTTL: 1m
# without unit -> use minutes
refreshPeriod: 120
clientLookup:
upstream: 192.168.178.1
singleNameOrder:
- 2
- 1

queryLog:
type: csv-client
target: /opt/log

port: 55553,:55554,[::1]:55555
logLevel: debug
dohUserAgent: testBlocky

0 comments on commit 0731ebe

Please sign in to comment.