Add prometheus reporting for replicas scaled (#1)

* Add prometheus reporting for replicas scaled * Bump rake version to address a CVE
hammady · Apr 26, 2020 · 3d13dd9 · 3d13dd9
1 parent a518e9a
commit 3d13dd9
Show file tree

Hide file tree

Showing 8 changed files with 162 additions and 12 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,7  6,6 @@
 /pkg/
 /spec/reports/
 /tmp/
-Gemfile.lock
 .env
 scaltainer.yml
 scaltainer.yml.state

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -0,0  1,104 @@
 PATH
   remote: .
   specs:
     scaltainer (0.3.0)
       docker-api
       dotenv
       excon (>= 0.47.0)
       kubeclient
       prometheus-client
 
 GEM
   remote: https://rubygems.org/
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     coderay (1.1.1)
     coveralls (0.8.21)
       json (>= 1.8, < 3)
       simplecov (~> 0.14.1)
       term-ansicolor (~> 1.3)
       thor (~> 0.19.4)
       tins (~> 1.6)
     diff-lcs (1.3)
     docile (1.1.5)
     docker-api (1.34.2)
       excon (>= 0.47.0)
       multi_json
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
     dotenv (2.7.5)
     excon (0.73.0)
     ffi (1.12.2)
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
     http (4.4.1)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
       http-form_data (~> 2.2)
       http-parser (~> 1.2.0)
     http-accept (1.7.0)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     http-form_data (2.3.0)
     http-parser (1.2.1)
       ffi-compiler (>= 1.0, < 2.0)
     json (2.1.0)
     kubeclient (4.6.0)
       http (>= 3.0, < 5.0)
       recursive-open-struct (~> 1.0, >= 1.0.4)
       rest-client (~> 2.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2020.0425)
     multi_json (1.14.1)
     netrc (0.11.0)
     prometheus-client (2.0.0)
     public_suffix (4.0.4)
     rake (13.0.1)
     recursive-open-struct (1.1.1)
     rest-client (2.1.0)
       http-accept (>= 1.7.0, < 2.0)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
     rspec (3.6.0)
       rspec-core (~> 3.6.0)
       rspec-expectations (~> 3.6.0)
       rspec-mocks (~> 3.6.0)
     rspec-core (3.6.0)
       rspec-support (~> 3.6.0)
     rspec-expectations (3.6.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.6.0)
     rspec-mocks (3.6.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.6.0)
     rspec-support (3.6.0)
     simplecov (0.14.1)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
     term-ansicolor (1.6.0)
       tins (~> 1.0)
     thor (0.19.4)
     tins (1.13.2)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundler (~> 1.15)
   coderay (~> 1.1)
   coveralls (~> 0.8)
   rake (>= 12.3.3)
   rspec (~> 3.5)
   scaltainer!
 
 BUNDLED WITH
    1.17.3
diff --git a/README.md b/README.md
@@ -55,6  55,21 @@ specify the wait time between repetitions using the `-w` parameter in seconds:
 
 This will repeatedly call scaltainer every 60 seconds, sleeping in-between.
 
 If you would like to monitor the changes in scaling out and in. You can install
 Prometheus and add a configuration parameter pointing to its Push Gateway:
 
     scaltainer -g prometheus-pushgateway.monitoring.svc.cluster.local:9091
 
 Where `prometheus-pushgateway.monitoring.svc.cluster.local:9091` is the address
 of the push gateway. For Kubernetes environments the above denotes the gateway service
 name (`prometheus-pushgateway`), where it is installed in the namespace called
 `monitoring`. Scaltainer will report the following metrics to Prometheus:
 
 - `rayyan_controller_replicas`: number of replicas scaled (or untouched thereof).
 This is labeled by the namespace and controller name, both matching the scaltainer
 configuration file.
 - `rayyan_scaltainer_ticks`: iterations scaltainer has performed (if `-w` is used)
 
 ## Configuration
 
 ### Environment variables
@@ -121,7  136,7 @@ The configuration file (determined by `-f FILE` command line parameter) should b
 
     # to get worker metrics
     endpoint: https://your-app.com/hirefire/$HIREFIRE_TOKEN/info
-    # optional docker swarm stack name or kubernetes namespace
     # optional docker swarm stack name or kubernetes namespace (useful if having push gateway)
     namespace: mynamespace
     # list of web services to monitor
     web_services:

diff --git a/exe/scaltainer b/exe/scaltainer
@@ -3,8  3,8 @@
 require 'scaltainer'
 
 begin
-  configfile, statefile, logger, wait, orchestrator = Scaltainer::Command.parse ARGV
-  Scaltainer::Runner.new configfile, statefile, logger, wait, orchestrator
   configfile, statefile, logger, wait, orchestrator, pushgateway = Scaltainer::Command.parse ARGV
   Scaltainer::Runner.new configfile, statefile, logger, wait, orchestrator, pushgateway
 rescue => e
   $stderr.puts e.message
   $stderr.puts e.backtrace

diff --git a/lib/scaltainer/command.rb b/lib/scaltainer/command.rb
@@ -4,7  4,7 @@
 module Scaltainer
   class Command
     def self.parse(args)
-      configfile, statefile, wait, orchestrator = 'scaltainer.yml', nil, 0, :swarm
       configfile, statefile, wait, orchestrator, pushgateway = 'scaltainer.yml', nil, 0, :swarm, nil
       OptionParser.new do |opts|
         opts.banner = "Usage: scaltainer [options]"
         opts.on("-f", "--conf-file FILE", "Specify configuration file (default: scaltainer.yml)") do |file|
@@ -19,6  19,9 @@ def self.parse(args)
         opts.on("-o", "--orchestrator swarm:kubernetes", [:swarm, :kubernetes], "Specify orchestrator type (default: swarm)") do |o|
           orchestrator = o
         end
         opts.on("-g", "--prometheus-push-gateway ADDRESS", "Specify prometheus push gateway address in the form of host:port") do |gw|
           pushgateway = gw
         end
         opts.on("-v", "--version", "Show version and exit") do
           puts Scaltainer::VERSION
           exit 0
@@ -55,7  58,7 @@ def self.parse(args)
       logger = Logger.new(STDOUT)
       logger.level = %w(debug info warn error fatal unknown).find_index((ENV['LOG_LEVEL'] || '').downcase) || 1
 
-      return configfile, statefile, logger, wait, orchestrator
       return configfile, statefile, logger, wait, orchestrator, pushgateway
     end
 
     private

diff --git a/lib/scaltainer/runner.rb b/lib/scaltainer/runner.rb
@@ -1,8  1,10 @@
 require "yaml"
 require 'prometheus/client'
 require 'prometheus/client/push'
 
 module Scaltainer
   class Runner
-    def initialize(configfile, statefile, logger, wait, orchestrator)
     def initialize(configfile, statefile, logger, wait, orchestrator, pushgateway)
       @orchestrator = orchestrator
       @logger = logger
       @default_service_config = {
@@ -19,18  21,20 @@ def initialize(configfile, statefile, logger, wait, orchestrator)
       endpoint = config["endpoint"]
       service_type_web = ServiceTypeWeb.new(endpoint)
       service_type_worker = ServiceTypeWorker.new(endpoint)
       register_pushgateway(pushgateway) if pushgateway
       namespace = config["namespace"] || config["stack_name"]
       loop do
-        run config, state, service_type_web, service_type_worker
         run config, state, service_type_web, service_type_worker, namespace
         save_state statefile, state
         sync_pushgateway(namespace, state) if pushgateway
         sleep wait
         break if wait == 0
       end
     end
 
     private
 
-    def run(config, state, service_type_web, service_type_worker)
-      namespace = config["namespace"] || config["stack_name"]
     def run(config, state, service_type_web, service_type_worker, namespace)
       iterate_services config["web_services"], namespace, service_type_web, state
       iterate_services config["worker_services"], namespace, service_type_worker, state
     end
@@ -82,9  86,11 @@ def process_service(service_name, config, state, namespace, type, metrics)
       adjusted_replicas = type.adjust_desired_replicas(desired_replicas, config)
       @logger.debug "Desired number of replicas for #{service.type} #{service.name} is adjusted to #{adjusted_replicas}"
       replica_diff = adjusted_replicas - current_replicas
       state["replicas"] = current_replicas
       type.yield_to_scale(replica_diff, config, state, metric,
         service.name, @logger) do
           scale_out service, current_replicas, adjusted_replicas
           state["replicas"] = adjusted_replicas
         end
     end
 
@@ -113,5  119,27 @@ def scale_out(service, current_replicas, desired_replicas)
       end
     end
 
     def register_pushgateway(pushgateway)
       @registry = Prometheus::Client.registry
       @replicas_gauge = @registry.gauge(:rayyan_controller_replicas, docstring: 'Rayyan replicas', labels: [:controller, :namespace])
       @ticks_counter = @registry.counter(:rayyan_scaltainer_ticks, docstring: 'Rayyan Scaltainer ticks', labels: [:namespace])
 
       @pushgateway = Prometheus::Client::Push.new("scaltainer", "scaltainer", "http://#{pushgateway}")
     end
 
     def sync_pushgateway(namespace, state)
       @logger.debug("Now syncing state #{state} in namespace #{namespace}")
       state.each do |service, state|
         @replicas_gauge.set(state["replicas"], labels: {namespace: namespace, controller: service}) if state["replicas"]
       end
       @ticks_counter.increment(labels: {namespace: namespace})
       begin
         @pushgateway.add(@registry)
       rescue => e
         @logger.warn "[#{e.class}] Error pushing metrics to the configured Prometheus Push Gateway: #{e.message}"
       end
       @logger.info "Pushed metrics successfully to the configured Prometheus Push Gateway"
     end
 
   end # class
 end # module
diff --git a/lib/scaltainer/version.rb b/lib/scaltainer/version.rb
@@ -1,3  1,3 @@
 module Scaltainer
-  VERSION = "0.2.0"
   VERSION = "0.3.0"
 end
diff --git a/scaltainer.gemspec b/scaltainer.gemspec
@@ -23,7  23,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 1.15"
-  spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rake", ">= 12.3.3"
   spec.add_development_dependency 'rspec', '~> 3.5'
   spec.add_development_dependency 'coderay', '~> 1.1'
   spec.add_development_dependency 'coveralls', '~> 0.8'
@@ -32,4  32,5 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "docker-api"
   spec.add_runtime_dependency "kubeclient"
   spec.add_runtime_dependency "dotenv"
   spec.add_runtime_dependency "prometheus-client"
 end