Coverage for lib/ansible/plugins/loader.py : 75%

# (c) 2012, Daniel Hokka Zakrisson <daniel@hozac.com> 

# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> and others 

# (c) 2017, Toshio Kuratomi <tkuratomi@ansible.com> 

# (c) 2017 Ansible Project 

# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) 

from __future__ import (absolute_import, division, print_function) 

__metaclass__ = type 

import glob 

import imp 

import os 

import os.path 

import sys 

import warnings 

from collections import defaultdict 

from ansible import constants as C 

from ansible.errors import AnsibleError 

from ansible.module_utils._text import to_text 

from ansible.parsing.utils.yaml import from_yaml 

from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_PATH_CACHE 

from ansible.utils.plugin_docs import get_docstring 

try: 

from __main__ import display 

except ImportError: 

from ansible.utils.display import Display 

display = Display() 

def get_all_plugin_loaders(): 

return [(name, obj) for (name, obj) in globals().items() if isinstance(obj, PluginLoader)] 

class PluginLoader: 

''' 

PluginLoader loads plugins from the configured plugin directories. 

It searches for plugins by iterating through the combined list of 

play basedirs, configured paths, and the python path. 

The first match is used. 

''' 

def __init__(self, class_name, package, config, subdir, aliases=None, required_base_class=None): 

aliases = {} if aliases is None else aliases 

self.class_name = class_name 

self.base_class = required_base_class 

self.package = package 

self.subdir = subdir 

# FIXME: remove alias dict in favor of alias by symlink? 

self.aliases = aliases 

if config and not isinstance(config, list): 

config = [config] 

60 ↛ 61line 60 didn't jump to line 61, because the condition on line 60 was never true elif not config: 

config = [] 

self.config = config 

if class_name not in MODULE_CACHE: 

MODULE_CACHE[class_name] = {} 

if class_name not in PATH_CACHE: 

PATH_CACHE[class_name] = None 

if class_name not in PLUGIN_PATH_CACHE: 

PLUGIN_PATH_CACHE[class_name] = defaultdict(dict) 

self._module_cache = MODULE_CACHE[class_name] 

self._paths = PATH_CACHE[class_name] 

self._plugin_path_cache = PLUGIN_PATH_CACHE[class_name] 

self._extra_dirs = [] 

self._searched_paths = set() 

def __setstate__(self, data): 

''' 

Deserializer. 

''' 

class_name = data.get('class_name') 

package = data.get('package') 

config = data.get('config') 

subdir = data.get('subdir') 

aliases = data.get('aliases') 

base_class = data.get('base_class') 

PATH_CACHE[class_name] = data.get('PATH_CACHE') 

PLUGIN_PATH_CACHE[class_name] = data.get('PLUGIN_PATH_CACHE') 

self.__init__(class_name, package, config, subdir, aliases, base_class) 

self._extra_dirs = data.get('_extra_dirs', []) 

self._searched_paths = data.get('_searched_paths', set()) 

def __getstate__(self): 

''' 

Serializer. 

''' 

return dict( 

class_name=self.class_name, 

base_class=self.base_class, 

package=self.package, 

config=self.config, 

subdir=self.subdir, 

aliases=self.aliases, 

_extra_dirs=self._extra_dirs, 

_searched_paths=self._searched_paths, 

PATH_CACHE=PATH_CACHE[self.class_name], 

PLUGIN_PATH_CACHE=PLUGIN_PATH_CACHE[self.class_name], 

) 

def format_paths(self, paths): 

''' Returns a string suitable for printing of the search path ''' 

# Uses a list to get the order right 

ret = [] 

for i in paths: 

122 ↛ 121line 122 didn't jump to line 121, because the condition on line 122 was never false if i not in ret: 

ret.append(i) 

return os.pathsep.join(ret) 

def print_paths(self): 

return self.format_paths(self._get_paths(subdirs=False)) 

def _all_directories(self, dir): 

results = [] 

results.append(dir) 

for root, subdirs, files in os.walk(dir, followlinks=True): 

if '__init__.py' in files: 

for x in subdirs: 

results.append(os.path.join(root, x)) 

return results 

def _get_package_paths(self, subdirs=True): 

''' Gets the path of a Python package ''' 

141 ↛ 142line 141 didn't jump to line 142, because the condition on line 141 was never true if not self.package: 

return [] 

143 ↛ 149line 143 didn't jump to line 149, because the condition on line 143 was never false if not hasattr(self, 'package_path'): 

m = __import__(self.package) 

parts = self.package.split('.')[1:] 

for parent_mod in parts: 

m = getattr(m, parent_mod) 

self.package_path = os.path.dirname(m.__file__) 

if subdirs: 

return self._all_directories(self.package_path) 

return [self.package_path] 

def _get_paths(self, subdirs=True): 

''' Return a list of paths to search for plugins in ''' 

# FIXME: This is potentially buggy if subdirs is sometimes True and sometimes False. 

# In current usage, everything calls this with subdirs=True except for module_utils_loader and ansible-doc 

# which always calls it with subdirs=False. So there currently isn't a problem with this caching. 

if self._paths is not None: 

return self._paths 

ret = self._extra_dirs[:] 

# look in any configured plugin paths, allow one level deep for subcategories 

165 ↛ 179line 165 didn't jump to line 179, because the condition on line 165 was never false if self.config is not None: 

for path in self.config: 

path = os.path.realpath(os.path.expanduser(path)) 

if subdirs: 

contents = glob.glob("%s/*" % path) + glob.glob("%s/*/*" % path) 

170 ↛ 171line 170 didn't jump to line 171, because the loop on line 170 never started for c in contents: 

if os.path.isdir(c) and c not in ret: 

ret.append(c) 

173 ↛ 166line 173 didn't jump to line 166, because the condition on line 173 was never false if path not in ret: 

ret.append(path) 

# look for any plugins installed in the package subtree 

# Note package path always gets added last so that every other type of 

# path is searched before it. 

ret.extend(self._get_package_paths(subdirs=subdirs)) 

# HACK: because powershell modules are in the same directory 

# hierarchy as other modules we have to process them last. This is 

# because powershell only works on windows but the other modules work 

# anywhere (possibly including windows if the correct language 

# interpreter is installed). the non-powershell modules can have any 

# file extension and thus powershell modules are picked up in that. 

# The non-hack way to fix this is to have powershell modules be 

# a different PluginLoader/ModuleLoader. But that requires changing 

# other things too (known thing to change would be PATHS_CACHE, 

# PLUGIN_PATHS_CACHE, and MODULE_CACHE. Since those three dicts key 

# on the class_name and neither regular modules nor powershell modules 

# would have class_names, they would not work as written. 

reordered_paths = [] 

win_dirs = [] 

for path in ret: 

if path.endswith('windows'): 

win_dirs.append(path) 

else: 

reordered_paths.append(path) 

reordered_paths.extend(win_dirs) 

# cache and return the result 

self._paths = reordered_paths 

return reordered_paths 

def _load_config_defs(self, name, path): 

''' Reads plugin docs to find configuration setting definitions, to push to config manager for later use ''' 

# plugins w/o class name don't support config 

211 ↛ exitline 211 didn't return from function '_load_config_defs', because the condition on line 211 was never false if self.class_name: 

type_name = get_plugin_class(self.class_name) 

# FIXME: expand to other plugins, but never doc fragments 

# if type name != 'module_doc_fragment': 

if type_name in ('callback', 'connection', 'inventory', 'lookup', 'shell'): 

dstring = get_docstring(path, fragment_loader, verbose=False, ignore_errors=True)[0] 

if dstring and 'options' in dstring and isinstance(dstring['options'], dict): 

C.config.initialize_plugin_configuration_definitions(type_name, name, dstring['options']) 

display.debug('Loaded config def from plugin (%s/%s)' % (type_name, name)) 

def add_directory(self, directory, with_subdir=False): 

''' Adds an additional directory to the search path ''' 

directory = os.path.realpath(directory) 

if directory is not None: 

if with_subdir: 

directory = os.path.join(directory, self.subdir) 

if directory not in self._extra_dirs: 

# append the directory and invalidate the path cache 

self._extra_dirs.append(directory) 

self._paths = None 

display.debug('Added %s to loader search path' % (directory)) 

def find_plugin(self, name, mod_type='', ignore_deprecated=False, check_aliases=False): 

''' Find a plugin named name ''' 

global _PLUGIN_FILTERS 

241 ↛ 242line 241 didn't jump to line 242, because the condition on line 241 was never true if name in _PLUGIN_FILTERS[self.package]: 

return None 

244 ↛ 245line 244 didn't jump to line 245, because the condition on line 244 was never true if mod_type: 

suffix = mod_type 

elif self.class_name: 

# Ansible plugins that run in the controller process (most plugins) 

suffix = '.py' 

else: 

# Only Ansible Modules. Ansible modules can be any executable so 

# they can have any suffix 

suffix = '' 

254 ↛ 255line 254 didn't jump to line 255, because the condition on line 254 was never true if check_aliases: 

name = self.aliases.get(name, name) 

# The particular cache to look for modules within. This matches the 

# requested mod_type 

pull_cache = self._plugin_path_cache[suffix] 

try: 

return pull_cache[name] 

except KeyError: 

# Cache miss. Now let's find the plugin 

pass 

# TODO: Instead of using the self._paths cache (PATH_CACHE) and 

# self._searched_paths we could use an iterator. Before enabling that 

# we need to make sure we don't want to add additional directories 

# (add_directory()) once we start using the iterator. Currently, it 

# looks like _get_paths() never forces a cache refresh so if we expect 

# additional directories to be added later, it is buggy. 

for path in (p for p in self._get_paths() if p not in self._searched_paths and os.path.isdir(p)): 

try: 

full_paths = (os.path.join(path, f) for f in os.listdir(path)) 

except OSError as e: 

display.warning("Error accessing plugin paths: %s" % to_text(e)) 

for full_path in (f for f in full_paths if os.path.isfile(f) and not f.endswith('__init__.py')): 

full_name = os.path.basename(full_path) 

# HACK: We have no way of executing python byte compiled files as ansible modules so specifically exclude them 

# FIXME: I believe this is only correct for modules and module_utils. 

# For all other plugins we want .pyc and .pyo should be valid 

if full_path.endswith(('.pyc', '.pyo')): 

continue 

splitname = os.path.splitext(full_name) 

base_name = splitname[0] 

try: 

extension = splitname[1] 

except IndexError: 

extension = '' 

# Module found, now enter it into the caches that match this file 

if base_name not in self._plugin_path_cache['']: 

self._plugin_path_cache[''][base_name] = full_path 

298 ↛ 301line 298 didn't jump to line 301, because the condition on line 298 was never false if full_name not in self._plugin_path_cache['']: 

self._plugin_path_cache[''][full_name] = full_path 

301 ↛ 304line 301 didn't jump to line 304, because the condition on line 301 was never false if base_name not in self._plugin_path_cache[extension]: 

self._plugin_path_cache[extension][base_name] = full_path 

304 ↛ 278line 304 didn't jump to line 278, because the condition on line 304 was never false if full_name not in self._plugin_path_cache[extension]: 

self._plugin_path_cache[extension][full_name] = full_path 

self._searched_paths.add(path) 

try: 

return pull_cache[name] 

except KeyError: 

# Didn't find the plugin in this directory. Load modules from the next one 

pass 

# if nothing is found, try finding alias/deprecated 

315 ↛ 325line 315 didn't jump to line 325, because the condition on line 315 was never false if not name.startswith('_'): 

alias_name = '_' + name 

# We've already cached all the paths at this point 

318 ↛ 319line 318 didn't jump to line 319, because the condition on line 318 was never true if alias_name in pull_cache: 

if not ignore_deprecated and not os.path.islink(pull_cache[alias_name]): 

# FIXME: this is not always the case, some are just aliases 

display.deprecated('%s is kept for backwards compatibility but usage is discouraged. ' 

'The module documentation details page may explain more about this rationale.' % name.lstrip('_')) 

return pull_cache[alias_name] 

return None 

def has_plugin(self, name): 

''' Checks if a plugin named name exists ''' 

return self.find_plugin(name) is not None 

__contains__ = has_plugin 

def _load_module_source(self, name, path): 

# avoid collisions across plugins 

full_name = '.'.join([self.package, name]) 

if full_name in sys.modules: 

# Avoids double loading, See https://github.com/ansible/ansible/issues/13110 

return sys.modules[full_name] 

with warnings.catch_warnings(): 

warnings.simplefilter("ignore", RuntimeWarning) 

with open(path, 'rb') as module_file: 

module = imp.load_source(full_name, path, module_file) 

return module 

def _update_object(self, obj, name, path): 

# load plugin config data 

self._load_config_defs(name, path) 

# set extra info on the module, in case we want it later 

setattr(obj, '_original_path', path) 

setattr(obj, '_load_name', name) 

def get(self, name, *args, **kwargs): 

''' instantiates a plugin of the given name using arguments ''' 

found_in_cache = True 

class_only = kwargs.pop('class_only', False) 

if name in self.aliases: 

name = self.aliases[name] 

path = self.find_plugin(name) 

if path is None: 

return None 

if path not in self._module_cache: 

self._module_cache[path] = self._load_module_source(name, path) 

found_in_cache = False 

obj = getattr(self._module_cache[path], self.class_name) 

if self.base_class: 

# The import path is hardcoded and should be the right place, 

# so we are not expecting an ImportError. 

module = __import__(self.package, fromlist=[self.base_class]) 

# Check whether this obj has the required base class. 

try: 

plugin_class = getattr(module, self.base_class) 

except AttributeError: 

return None 

383 ↛ 384line 383 didn't jump to line 384, because the condition on line 383 was never true if not issubclass(obj, plugin_class): 

return None 

self._display_plugin_load(self.class_name, name, self._searched_paths, path, found_in_cache=found_in_cache, class_only=class_only) 

if not class_only: 

try: 

obj = obj(*args, **kwargs) 

except TypeError as e: 

if "abstract" in e.args[0]: 

# Abstract Base Class. The found plugin file does not 

# fully implement the defined interface. 

return None 

raise 

self._update_object(obj, name, path) 

return obj 

def _display_plugin_load(self, class_name, name, searched_paths, path, found_in_cache=None, class_only=None): 

msg = 'Loading %s \'%s\' from %s' % (class_name, os.path.basename(name), path) 

if len(searched_paths) > 1: 

msg = '%s (searched paths: %s)' % (msg, self.format_paths(searched_paths)) 

if found_in_cache or class_only: 

msg = '%s (found_in_cache=%s, class_only=%s)' % (msg, found_in_cache, class_only) 

display.debug(msg) 

def all(self, *args, **kwargs): 

''' instantiates all plugins with the same arguments ''' 

global _PLUGIN_FILTERS 

path_only = kwargs.pop('path_only', False) 

class_only = kwargs.pop('class_only', False) 

all_matches = [] 

found_in_cache = True 

for i in self._get_paths(): 

all_matches.extend(glob.glob(os.path.join(i, "*.py"))) 

for path in sorted(all_matches, key=os.path.basename): 

name = os.path.basename(os.path.splitext(path)[0]) 

427 ↛ 428line 427 didn't jump to line 428, because the condition on line 427 was never true if '__init__' in name or name in _PLUGIN_FILTERS[self.package]: 

continue 

if path_only: 

yield path 

continue 

if path not in self._module_cache: 

module = self._load_module_source(name, path) 

436 ↛ 441line 436 didn't jump to line 441, because the condition on line 436 was never true if module in self._module_cache.values(): 

# In ``_load_module_source`` if a plugin has a duplicate name, we just return the 

# previously matched plugin from sys.modules, which means you are never getting both, 

# just one, but cached for both paths, this isn't normally a problem, except with callbacks 

# where it will run that single callback twice. This rejects duplicates. 

continue 

self._module_cache[path] = module 

found_in_cache = False 

try: 

obj = getattr(self._module_cache[path], self.class_name) 

except AttributeError as e: 

display.warning("Skipping plugin (%s) as it seems to be invalid: %s" % (path, to_text(e))) 

continue 

451 ↛ 454line 451 didn't jump to line 454, because the condition on line 451 was never true if self.base_class: 

# The import path is hardcoded and should be the right place, 

# so we are not expecting an ImportError. 

module = __import__(self.package, fromlist=[self.base_class]) 

# Check whether this obj has the required base class. 

try: 

plugin_class = getattr(module, self.base_class) 

except AttributeError: 

continue 

if not issubclass(obj, plugin_class): 

continue 

self._display_plugin_load(self.class_name, name, self._searched_paths, path, found_in_cache=found_in_cache, class_only=class_only) 

if not class_only: 

try: 

obj = obj(*args, **kwargs) 

except TypeError as e: 

display.warning("Skipping plugin (%s) as it seems to be incomplete: %s" % (path, to_text(e))) 

self._update_object(obj, name, path) 

yield obj 

def _load_plugin_filter(): 

filters = defaultdict(frozenset) 

477 ↛ 481line 477 didn't jump to line 481, because the condition on line 477 was never false if C.PLUGIN_FILTERS_CFG is None: 

filter_cfg = '/etc/ansible/plugin_filters.yml' 

user_set = False 

else: 

filter_cfg = C.PLUGIN_FILTERS_CFG 

user_set = True 

484 ↛ 485line 484 didn't jump to line 485, because the condition on line 484 was never true if os.path.exists(filter_cfg): 

with open(filter_cfg, 'rb') as f: 

try: 

filter_data = from_yaml(f.read()) 

except Exception as e: 

display.warning(u'The plugin filter file, {0} was not parsable.' 

u' Skipping: {1}'.format(filter_cfg, to_text(e))) 

return filters 

try: 

version = filter_data['filter_version'] 

except KeyError: 

display.warning(u'The plugin filter file, {0} was invalid.' 

u' Skipping.'.format(filter_cfg)) 

return filters 

# Try to convert for people specifying version as a float instead of string 

version = to_text(version) 

version = version.strip() 

if version == u'1.0': 

# Modules and action plugins share the same blacklist since the difference between the 

# two isn't visible to the users 

filters['ansible.modules'] = frozenset(filter_data['module_blacklist']) 

filters['ansible.plugins.action'] = filters['ansible.modules'] 

else: 

display.warning(u'The plugin filter file, {0} was a version not recognized by this' 

u' version of Ansible. Skipping.') 

else: 

513 ↛ 514line 513 didn't jump to line 514, because the condition on line 513 was never true if user_set: 

display.warning(u'The plugin filter file, {0} does not exist.' 

u' Skipping.'.format(filter_cfg)) 

# Specialcase the stat module as Ansible can run very few things if stat is blacklisted. 

518 ↛ 519line 518 didn't jump to line 519, because the condition on line 518 was never true if 'stat' in filters['ansible.modules']: 

raise AnsibleError('The stat module was specified in the module blacklist file, {0}, but' 

' Ansible will not function without the stat module. Please remove stat' 

' from the blacklist.'.format(filter_cfg)) 

return filters 

# TODO: All of the following is initialization code It should be moved inside of an initialization 

# function which is called at some point early in the ansible and ansible-playbook CLI startup. 

_PLUGIN_FILTERS = _load_plugin_filter() 

# doc fragments first 

fragment_loader = PluginLoader( 

'ModuleDocFragment', 

'ansible.utils.module_docs_fragments', 

os.path.join(os.path.dirname(__file__), 'module_docs_fragments'), 

'', 

) 

action_loader = PluginLoader( 

'ActionModule', 

'ansible.plugins.action', 

C.DEFAULT_ACTION_PLUGIN_PATH, 

'action_plugins', 

required_base_class='ActionBase', 

) 

cache_loader = PluginLoader( 

'CacheModule', 

'ansible.plugins.cache', 

C.DEFAULT_CACHE_PLUGIN_PATH, 

'cache_plugins', 

) 

callback_loader = PluginLoader( 

'CallbackModule', 

'ansible.plugins.callback', 

C.DEFAULT_CALLBACK_PLUGIN_PATH, 

'callback_plugins', 

) 

connection_loader = PluginLoader( 

'Connection', 

'ansible.plugins.connection', 

C.DEFAULT_CONNECTION_PLUGIN_PATH, 

'connection_plugins', 

aliases={'paramiko': 'paramiko_ssh'}, 

required_base_class='ConnectionBase', 

) 

shell_loader = PluginLoader( 

'ShellModule', 

'ansible.plugins.shell', 

'shell_plugins', 

'shell_plugins', 

) 

module_loader = PluginLoader( 

'', 

'ansible.modules', 

C.DEFAULT_MODULE_PATH, 

'library', 

) 

module_utils_loader = PluginLoader( 

'', 

'ansible.module_utils', 

C.DEFAULT_MODULE_UTILS_PATH, 

'module_utils', 

) 

# NB: dedicated loader is currently necessary because PS module_utils expects "with subdir" lookup where 

# regular module_utils doesn't. This can be revisited once we have more granular loaders. 

ps_module_utils_loader = PluginLoader( 

'', 

'ansible.module_utils', 

C.DEFAULT_MODULE_UTILS_PATH, 

'module_utils', 

) 

lookup_loader = PluginLoader( 

'LookupModule', 

'ansible.plugins.lookup', 

C.DEFAULT_LOOKUP_PLUGIN_PATH, 

'lookup_plugins', 

required_base_class='LookupBase', 

) 

filter_loader = PluginLoader( 

'FilterModule', 

'ansible.plugins.filter', 

C.DEFAULT_FILTER_PLUGIN_PATH, 

'filter_plugins', 

) 

test_loader = PluginLoader( 

'TestModule', 

'ansible.plugins.test', 

C.DEFAULT_TEST_PLUGIN_PATH, 

'test_plugins' 

) 

strategy_loader = PluginLoader( 

'StrategyModule', 

'ansible.plugins.strategy', 

C.DEFAULT_STRATEGY_PLUGIN_PATH, 

'strategy_plugins', 

required_base_class='StrategyBase', 

) 

terminal_loader = PluginLoader( 

'TerminalModule', 

'ansible.plugins.terminal', 

'terminal_plugins', 

'terminal_plugins' 

) 

vars_loader = PluginLoader( 

'VarsModule', 

'ansible.plugins.vars', 

C.DEFAULT_VARS_PLUGIN_PATH, 

'vars_plugins', 

) 

cliconf_loader = PluginLoader( 

'Cliconf', 

'ansible.plugins.cliconf', 

'cliconf_plugins', 

'cliconf_plugins', 

required_base_class='CliconfBase' 

) 

netconf_loader = PluginLoader( 

'Netconf', 

'ansible.plugins.netconf', 

'netconf_plugins', 

'netconf_plugins', 

required_base_class='NetconfBase' 

) 

inventory_loader = PluginLoader( 

'InventoryModule', 

'ansible.plugins.inventory', 

C.DEFAULT_INVENTORY_PLUGIN_PATH, 

'inventory_plugins' 

)