set config on everywhere to allow for future simulator testing (#1076)

* Use a user defined url * Start refactoring the simulator to be more comprehensive * Remove breakpoint * Add graphql to simulator * Add more graphql endpoints * Fix cache dir pathing for repowrapper * refator to class and perist data to disk * Add config option to disable ratelimiting * initial handling of pullrequests * Handle NoneType version strings * Use ratelimit config in webscraper * Show unknown routes * Handle stringified numbers in summary indexes * Use the url provided by the PR's head * Use username in head url to fake a forked path * refactor pullrequest getter * cleanup * set team to None to reduce calls * refactor more into mock * check for None instead of not * Add ability to load the bots pickle files as fixtures * more work on sim * prefix shippable cache with global cache * Load the files pickles * Add subcommands to fetch,load,generate * Fetch and load commits vs. pull commits * Fetch and load the 'git' commits too * Allow the indexers to use a specific commit * use the real comments * Use the context for shippable instead of checking url * Use a property for commits instead of getter * Fix bad assumption about matching events with no actor * Add token arg and use it for requests * Skip files that don't exist * Move flask script * WIP test script * Make the shippable url configurable * Use configurable shippable urls * update sim script * handle the mergeable state flag being unknown * Make sure non-ists are handled * end-to-end prototype working * Update vagrantfile for libvirt * some updates * compress the fixtures * WIP checkin * checkin * checkin before rebase * checkin * remove the test container files * fix some unicode "problems" * fix some more unicode * put logzero in constraints * temp fix for test * remove dead code
ansible · Apr 9, 2019 · 6c18235 · 6c18235
1 parent 92d09a6
commit 6c18235
Show file tree

Hide file tree

Showing 22 changed files with 538 additions and 257 deletions.
diff --git a/Vagrantfile b/Vagrantfile
@@ -14,7  14,7 @@ if [[ $RC != 0 ]]; then
 fi
 
 # BASELINE PACKAGES
-PACKAGES="epel-release ansible git vim-enhanced bind-utils policycoreutils-python net-tools lsof"
 PACKAGES="epel-release ansible git rsync vim-enhanced bind-utils policycoreutils-python net-tools lsof"
 for PKG in $PACKAGES; do
     rpm -q $PKG || yum -y install $PKG
 done
@@ -23,23  23,39 @@ done
 rm -f /etc/vimrc
 cp /vagrant/playbooks/files/centos7.vimrc /etc/vimrc
 
 # WORKAROUNDS
 setenforce 0
 fgrep docker /etc/group || groupadd -g 993 docker
 fgrep ansibot /etc/group || groupadd -g 1099 ansibot
 id ansibot || useradd -u 1099 -g ansibot ansibot
 usermod -a -G docker ansibot
 rsync -avz --exclude='/vagrant/.vagrant' /vagrant/* /home/ansibot/ansibullbot
 
 # PSEUDO ANSIBLE-LOCAL PROVISIONER
 setenforce 0
 echo "ansibullbot ansible_host=localhost ansible_connection=local" > /tmp/inv.ini
 echo "ansibullbot.eng.ansible.com ansible_host=localhost ansible_connection=local" >> /tmp/inv.ini
 cd /vagrant/playbooks
-ansible-playbook \
-    -v \
-    -i /tmp/inv.ini \
-    -e "ansibullbot_action=install" \
-    --skip-tags=botinstance,dns,ssh,ansibullbot_service,ansibullbot_logs \
-    setup-ansibullbot.yml
 #PLAYBOOKS="setup-ansibullbot.yml"
 PLAYBOOKS="vagrant.yml"
 for PLAYBOOK in $PLAYBOOKS; do
     ansible-playbook \
         -v \
         -i /tmp/inv.ini \
         --skip-tags=ssh \
         $PLAYBOOK
 done
 
 #        --tags=packages,ansibullbot,caddy \
 #        --skip-tags=botinstance,dns,ssh,ansibullbot_service,ansibullbot_logs \
 #--skip-tags=botinstance,dns,ssh,ansibullbot_service,ansibullbot_logs \
 #    -e "ansibullbot_action=install" \
 # HACK IN FIREWALL EXCEPTIONS
 firewall-cmd --zone=public --add-port=80/tcp --permanent
 firewall-cmd --reload
 
 SCRIPT
 
-ENV['VAGRANT_DEFAULT_PROVIDER'] = 'virtualbox'
 
 Vagrant.configure("2") do |config|
   config.vm.box = "centos/7"
@@ -49,7  65,13 @@ Vagrant.configure("2") do |config|
   config.hostmanager.manage_guest = true
   config.hostmanager.ignore_private_ip = false
   config.hostmanager.include_offline = true
-  config.vm.network "private_network", ip: "192.168.10.199"
-  config.vm.synced_folder ".", "/vagrant", type: "nfs"
   config.vm.network "private_network", ip: "10.0.0.210"
   config.vm.synced_folder ".", "/vagrant", type: "nfs", nfs_udp: false
 
   config.vm.provider :libvirt do |libvirt|
     libvirt.cpus = 2
     libvirt.memory = 2048
   end
 
   config.vm.provision "shell", inline: $script
 end
diff --git a/ansibullbot/constants.py b/ansibullbot/constants.py
@@ -220,6  220,16 @@ def load_config_file():
     value_type='boolean'
 )
 
 # Use or don't use the ratelimiting decorator
 DEFAULT_RATELIMIT = get_config(
     p,
     DEFAULTS,
     'ratelimit',
     '%s_RATELIMIT' % PROG_NAME.upper(),
     True,
     value_type='boolean'
 )
 
 DEFAULT_GITHUB_URL = get_config(
     p,
     DEFAULTS,
@@ -265,6  275,15 @@ def load_config_file():
     value_type='string'
 )
 
 DEFAULT_SHIPPABLE_URL = get_config(
     p,
     DEFAULTS,
     'shippable_url',
     '%s_SHIPPABLE_URL' % PROG_NAME.upper(),
     u'https://api.shippable.com',
     value_type='string'
 )
 
 DEFAULT_NEEDS_INFO_WARN = get_config(
     p,
     'needs_info',
@@ -293,6  312,7 @@ def load_config_file():
     value_type='int'
 )
 
 
 ###########################################
 #   METADATA RECEIVER
 ###########################################

diff --git a/ansibullbot/decorators/github.py b/ansibullbot/decorators/github.py
@@ -97,6  97,10 @@ def RateLimited(fn):
 
     def inner(*args, **kwargs):
 
         # bypass this decorator for testing purposes
         if not C.DEFAULT_RATELIMIT:
             return fn(*args, **kwargs)
 
         success = False
         count = 0
         while not success:

diff --git a/ansibullbot/triagers/ansible.py b/ansibullbot/triagers/ansible.py
@@ -183,11  183,11 @@ def __init__(self):
         super(AnsibleTriage, self).__init__()
 
         # get valid labels
-        logging.info('getting labels')
         logging.info(u'getting labels')
         self.valid_labels = self.get_valid_labels(u"ansible/ansible")
 
         self._ansible_members = []
-        self._ansible_core_team = []
         self._ansible_core_team = None
         self._botmeta_content = None
         self.botmeta = {}
         self.automerge_on = False
@@ -201,27  201,35 @@ def __init__(self):
         self.issue_summaries = {}
 
         # create the scraper for www data
-        logging.info('creating webscraper')
-        self.gws = GithubWebScraper(cachedir=self.cachedir_base)
         logging.info(u'creating webscraper')
         self.gws = GithubWebScraper(
             cachedir=self.cachedir_base,
             server=C.DEFAULT_GITHUB_URL
         )
         if C.DEFAULT_GITHUB_TOKEN:
-            self.gqlc = GithubGraphQLClient(C.DEFAULT_GITHUB_TOKEN)
             logging.info(u'creating graphql client')
             self.gqlc = GithubGraphQLClient(
                 C.DEFAULT_GITHUB_TOKEN,
                 server=C.DEFAULT_GITHUB_URL
             )
         else:
             self.gqlc = None
 
         # clone ansible/ansible
         repo = u'https://github.com/ansible/ansible'
-        gitrepo = GitRepoWrapper(cachedir=self.cachedir_base, repo=repo)
         gitrepo = GitRepoWrapper(cachedir=self.cachedir_base, repo=repo, commit=self.ansible_commit)
 
         # set the indexers
         logging.info('creating version indexer')
         self.version_indexer = AnsibleVersionIndexer(
-            checkoutdir=gitrepo.checkoutdir
             checkoutdir=gitrepo.checkoutdir,
             commit=self.ansible_commit
         )
 
         logging.info('creating file indexer')
         self.file_indexer = FileIndexer(
             botmetafile=self.botmetafile,
-            gitrepo=gitrepo
             gitrepo=gitrepo,
         )
 
         logging.info('creating module indexer')
@@ -237,12  245,12 @@ def __init__(self):
         self.component_matcher = AnsibleComponentMatcher(
             gitrepo=gitrepo,
             botmetafile=self.botmetafile,
-            email_cache=self.module_indexer.emails_cache
             email_cache=self.module_indexer.emails_cache,
         )
 
         # instantiate shippable api
         logging.info('creating shippable wrapper')
-        spath = os.path.expanduser(u'~/.ansibullbot/cache/shippable.runs')
         spath = os.path.join(self.cachedir_base, 'shippable.runs')
         self.SR = ShippableRuns(cachedir=spath, writecache=True)
         self.SR.update()
 
@@ -266,7  274,7 @@ def ansible_members(self):
 
     @property
     def ansible_core_team(self):
-        if not self._ansible_core_team:
         if self._ansible_core_team is None:
             teams = [
                 u'ansible-commit',
                 u'ansible-community',
@@ -1668,7  1676,7 @@ def collect_repos(self):
         self.gh = self._connect()
 
         logging.info('creating github connection wrapper')
-        self.ghw = GithubWrapper(self.gh)
         self.ghw = GithubWrapper(self.gh, cachedir=self.cachedir_base)
 
         for repo in REPOS:
             # skip repos based on args
@@ -2443,6  2451,9 @@ def create_parser(cls):
         parser.add_argument("--no_since", action="store_true",
                             help="Do not use the since keyword to fetch issues")
 
         parser.add_argument('--commit', dest='ansible_commit',
                             help="Use a specific commit for the indexers")
 
         return parser
 
     def get_resume(self):

diff --git a/ansibullbot/triagers/plugins/needs_revision.py b/ansibullbot/triagers/plugins/needs_revision.py
@@ -119,7  119,7 @@ def get_needs_revision_facts(triager, issuewrapper, meta, shippable=None):
             if u'travis-ci.org' in x[u'target_url']:
                 has_travis = True
                 continue
-            if u'shippable.com' in x[u'target_url']:
             if x.get('context') == 'Shippable':
                 has_shippable = True
                 continue
 
@@ -133,7  133,7 @@ def get_needs_revision_facts(triager, issuewrapper, meta, shippable=None):
         has_zuul = True
 
     ci_states = [x[u'state'] for x in ci_status
-                 if isinstance(x, dict) and u'shippable.com' in x[u'target_url']]
                  if isinstance(x, dict) and x.get('context') == 'Shippable']
 
     if not ci_states:
         ci_state = None

diff --git a/ansibullbot/triagers/plugins/small_patch.py b/ansibullbot/triagers/plugins/small_patch.py
@@ -19,7  19,7 @@ def get_small_patch_facts(iw):
 
     small_chunks_changed = 0
 
-    for commit in iw.get_commits():
     for commit in iw.commits:
         if iw.get_commit_files(commit) is None:
             # "Sorry, this diff is temporarily unavailable due to heavy server load."
             return sfacts

diff --git a/ansibullbot/utils/component_tools.py b/ansibullbot/utils/component_tools.py
@@ -104,14  104,15 @@ class AnsibleComponentMatcher(object):
         u'winrm': u'lib/ansible/plugins/connection/winrm.py'
     }
 
-    def __init__(self, gitrepo=None, botmetafile=None, cachedir=None, email_cache=None, file_indexer=None):
     def __init__(self, gitrepo=None, botmetafile=None, cachedir=None, commit=None, email_cache=None, file_indexer=None):
         self.botmetafile = botmetafile
         self.email_cache = email_cache
         self.commit = commit
 
         if gitrepo:
             self.gitrepo = gitrepo
         else:
-            self.gitrepo = GitRepoWrapper(cachedir=cachedir, repo=self.REPO)
             self.gitrepo = GitRepoWrapper(cachedir=cachedir, repo=self.REPO, commit=self.commit)
 
         if file_indexer:
             self.file_indexer = file_indexer
@@ -149,6  150,8 @@ def index_files(self):
         for fn in self.gitrepo.module_files:
             if os.path.isdir(fn):
                 continue
             if not os.path.exists(fn):
                 continue
             mname = os.path.basename(fn)
             mname = mname.replace(u'.py', u'').replace(u'.ps1', u'')
             if mname.startswith(u'__'):

diff --git a/ansibullbot/utils/extractors.py b/ansibullbot/utils/extractors.py
@@ -559,6  559,9 @@ def extract_github_id(self, author):
 
         authors = set()
 
         if author is None:
             return authors
 
         if u'ansible core team' in author.lower():
             authors.add(u'ansible')
         elif u'@' in author:

diff --git a/ansibullbot/utils/file_tools.py b/ansibullbot/utils/file_tools.py
@@ -31,13  31,14 @@ class FileIndexer(ModuleIndexer):
 
     files = []
 
-    def __init__(self, botmetafile=None, gitrepo=None):
     def __init__(self, botmetafile=None, gitrepo=None, commit=None):
         self.botmetafile = botmetafile
         self.botmeta = {}
         self.CMAP = {}
         self.FILEMAP = {}
         self.match_cache = {}
         self.gitrepo = gitrepo
         self.commit = commit
         self.update(force=True)
         self.email_commits = {}
 

diff --git a/ansibullbot/utils/gh_gql_client.py b/ansibullbot/utils/gh_gql_client.py
@@ -91,7  91,10 @@
 class GithubGraphQLClient(object):
     baseurl = u'https://api.github.com/graphql'
 
-    def __init__(self, token):
     def __init__(self, token, server=None):
         if server:
             # this is for testing
             self.baseurl = server.rstrip('/')   '/graphql'
         self.token = token
         self.headers = {
             u'Accept': u'application/json',

diff --git a/ansibullbot/utils/git_tools.py b/ansibullbot/utils/git_tools.py
@@ -13,8  13,9 @@ class GitRepoWrapper(object):
 
     _files = []
 
-    def __init__(self, cachedir, repo):
     def __init__(self, cachedir, repo, commit=None):
         self.repo = repo
         self.commit = commit
         self.checkoutdir = cachedir or u'~/.ansibullbot/cache'
         self.checkoutdir = os.path.join(cachedir, u'ansible.checkout')
         self.checkoutdir = os.path.expanduser(self.checkoutdir)
@@ -38,6  39,8 @@ def create_checkout(self):
             shutil.rmtree(self.checkoutdir)
         cmd = "git clone %s %s" \
             % (self.repo, self.checkoutdir)
         #if self.commit:
         #    import epdb; epdb.st()
         (rc, so, se) = run_command(cmd)
         print(to_text(so)   to_text(se))
 
@@ -53,19  56,37 @@ def update_checkout(self):
 
         changed = False
 
-        cmd = "cd %s ; git pull --rebase" % self.checkoutdir
-        (rc, so, se) = run_command(cmd)
-        so = to_text(so)
-        print(so   to_text(se))
         # get a specific commit or do a rebase
         if self.commit:
             cmd = "cd %s; git log -1  | head -n1 | awk '{print $2}'" % self.checkoutdir
             (rc, so, se) = run_command(cmd)
             so = to_text(so).strip()
 
-        # If rebase failed, recreate the checkout
-        if rc != 0:
-            self.create_checkout()
-            return True
-        else:
-            if u'current branch devel is up to date.' not in so.lower():
             if so != self.commit:
                 cmd = "cd %s; git checkout %s" % (self.checkoutdir, self.commit)
                 (rc, so, se) = run_command(cmd)
                 changed = True
 
             if rc != 0:
                 self.create_checkout()
                 changed = True
 
         else:
             changed = False
 
             cmd = "cd %s ; git pull --rebase" % self.checkoutdir
             (rc, so, se) = run_command(cmd)
             so = to_text(so)
             print(so   to_text(se))
 
             # If rebase failed, recreate the checkout
             if rc != 0:
                 self.create_checkout()
                 return True
             else:
                 if u'current branch devel is up to date.' not in so.lower():
                     changed = True
 
         self.commits_by_email = None
 
         return changed