Coverage for lib/ansible/parsing/dataloader.py : 57%

# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com> 

# Copyright: (c) 2017, Ansible Project 

# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) 

# Make coding more python3-ish 

from __future__ import (absolute_import, division, print_function) 

__metaclass__ = type 

import copy 

import os 

import os.path 

import re 

import tempfile 

from ansible.errors import AnsibleFileNotFound, AnsibleParserError 

from ansible.module_utils.basic import is_executable 

from ansible.module_utils.six import binary_type, text_type 

from ansible.module_utils._text import to_bytes, to_native, to_text 

from ansible.parsing.quoting import unquote 

from ansible.parsing.utils.yaml import from_yaml 

from ansible.parsing.vault import VaultLib, b_HEADER, is_encrypted, is_encrypted_file, parse_vaulttext_envelope 

from ansible.utils.path import unfrackpath 

try: 

from __main__ import display 

except ImportError: 

from ansible.utils.display import Display 

display = Display() 

# Tries to determine if a path is inside a role, last dir must be 'tasks' 

# this is not perfect but people should really avoid 'tasks' dirs outside roles when using Ansible. 

RE_TASKS = re.compile(u'(?:^|%s)+tasks%s?$' % (os.path.sep, os.path.sep)) 

class DataLoader: 

''' 

The DataLoader class is used to load and parse YAML or JSON content, 

either from a given file name or from a string that was previously 

read in through other means. A Vault password can be specified, and 

any vault-encrypted files will be decrypted. 

Data read from files will also be cached, so the file will never be 

read from disk more than once. 

Usage: 

dl = DataLoader() 

# optionally: dl.set_vault_password('foo') 

ds = dl.load('...') 

ds = dl.load_from_file('/path/to/file') 

''' 

def __init__(self): 

self._basedir = '.' 

self._FILE_CACHE = dict() 

self._tempfiles = set() 

# initialize the vault stuff with an empty password 

# TODO: replace with a ref to something that can get the password 

# a creds/auth provider 

# self.set_vault_password(None) 

self._vaults = {} 

self._vault = VaultLib() 

self.set_vault_secrets(None) 

# TODO: since we can query vault_secrets late, we could provide this to DataLoader init 

def set_vault_secrets(self, vault_secrets): 

self._vault.secrets = vault_secrets 

def load(self, data, file_name='<string>', show_content=True): 

'''Backwards compat for now''' 

return from_yaml(data, file_name, show_content, self._vault.secrets) 

def load_from_file(self, file_name, cache=True, unsafe=False): 

''' Loads data from a file, which can contain either JSON or YAML. ''' 

file_name = self.path_dwim(file_name) 

display.debug("Loading data from %s" % file_name) 

# if the file has already been read in and cached, we'll 

# return those results to avoid more file/vault operations 

if cache and file_name in self._FILE_CACHE: 

parsed_data = self._FILE_CACHE[file_name] 

else: 

# read the file contents and load the data structure from them 

(b_file_data, show_content) = self._get_file_contents(file_name) 

file_data = to_text(b_file_data, errors='surrogate_or_strict') 

parsed_data = self.load(data=file_data, file_name=file_name, show_content=show_content) 

# cache the file contents for next time 

self._FILE_CACHE[file_name] = parsed_data 

96 ↛ 97line 96 didn't jump to line 97, because the condition on line 96 was never true if unsafe: 

return parsed_data 

else: 

# return a deep copy here, so the cache is not affected 

return copy.deepcopy(parsed_data) 

def path_exists(self, path): 

path = self.path_dwim(path) 

return os.path.exists(to_bytes(path, errors='surrogate_or_strict')) 

def is_file(self, path): 

path = self.path_dwim(path) 

return os.path.isfile(to_bytes(path, errors='surrogate_or_strict')) or path == os.devnull 

def is_directory(self, path): 

path = self.path_dwim(path) 

return os.path.isdir(to_bytes(path, errors='surrogate_or_strict')) 

def list_directory(self, path): 

path = self.path_dwim(path) 

return os.listdir(path) 

def is_executable(self, path): 

'''is the given path executable?''' 

path = self.path_dwim(path) 

return is_executable(path) 

def _decrypt_if_vault_data(self, b_vault_data, b_file_name=None): 

'''Decrypt b_vault_data if encrypted and return b_data and the show_content flag''' 

126 ↛ 130line 126 didn't jump to line 130, because the condition on line 126 was never false if not is_encrypted(b_vault_data): 

show_content = True 

return b_vault_data, show_content 

b_ciphertext, b_version, cipher_name, vault_id = parse_vaulttext_envelope(b_vault_data) 

b_data = self._vault.decrypt(b_vault_data, filename=b_file_name) 

show_content = False 

return b_data, show_content 

def _get_file_contents(self, file_name): 

''' 

Reads the file contents from the given file name 

If the contents are vault-encrypted, it will decrypt them and return 

the decrypted data 

:arg file_name: The name of the file to read. If this is a relative 

path, it will be expanded relative to the basedir 

:raises AnsibleFileNotFOund: if the file_name does not refer to a file 

:raises AnsibleParserError: if we were unable to read the file 

:return: Returns a byte string of the file contents 

''' 

149 ↛ 150line 149 didn't jump to line 150, because the condition on line 149 was never true if not file_name or not isinstance(file_name, (binary_type, text_type)): 

raise AnsibleParserError("Invalid filename: '%s'" % str(file_name)) 

b_file_name = to_bytes(self.path_dwim(file_name)) 

# This is what we really want but have to fix unittests to make it pass 

# if not os.path.exists(b_file_name) or not os.path.isfile(b_file_name): 

155 ↛ 156line 155 didn't jump to line 156, because the condition on line 155 was never true if not self.path_exists(b_file_name) or not self.is_file(b_file_name): 

raise AnsibleFileNotFound("Unable to retrieve file contents", file_name=file_name) 

try: 

with open(b_file_name, 'rb') as f: 

data = f.read() 

return self._decrypt_if_vault_data(data, b_file_name) 

except (IOError, OSError) as e: 

raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (file_name, str(e)), orig_exc=e) 

def get_basedir(self): 

''' returns the current basedir ''' 

return self._basedir 

def set_basedir(self, basedir): 

''' sets the base directory, used to find files when a relative path is given ''' 

173 ↛ exitline 173 didn't return from function 'set_basedir', because the condition on line 173 was never false if basedir is not None: 

self._basedir = to_text(basedir) 

def path_dwim(self, given): 

''' 

make relative paths work like folks expect. 

''' 

given = unquote(given) 

given = to_text(given, errors='surrogate_or_strict') 

if given.startswith(to_text(os.path.sep)) or given.startswith(u'~'): 

path = given 

else: 

basedir = to_text(self._basedir, errors='surrogate_or_strict') 

path = os.path.join(basedir, given) 

return unfrackpath(path, follow=False) 

def _is_role(self, path): 

''' imperfect role detection, roles are still valid w/o tasks|meta/main.yml|yaml|etc ''' 

b_path = to_bytes(path, errors='surrogate_or_strict') 

b_upath = to_bytes(unfrackpath(path, follow=False), errors='surrogate_or_strict') 

198 ↛ 210line 198 didn't jump to line 210, because the loop on line 198 didn't complete for b_finddir in (b'meta', b'tasks'): 

199 ↛ 198line 199 didn't jump to line 198, because the loop on line 199 didn't complete for b_suffix in (b'.yml', b'.yaml', b''): 

b_main = b'main%s' % (b_suffix) 

b_tasked = os.path.join(b_finddir, b_main) 

203 ↛ 199line 203 didn't jump to line 199 if ( 

RE_TASKS.search(path) and 

os.path.exists(os.path.join(b_path, b_main)) or 

os.path.exists(os.path.join(b_upath, b_tasked)) or 

os.path.exists(os.path.join(os.path.dirname(b_path), b_tasked)) 

): 

return True 

return False 

def path_dwim_relative(self, path, dirname, source, is_role=False): 

''' 

find one file in either a role or playbook dir with or without 

explicitly named dirname subdirs 

Used in action plugins and lookups to find supplemental files that 

could be in either place. 

''' 

search = [] 

source = to_text(source, errors='surrogate_or_strict') 

# I have full path, nothing else needs to be looked at 

if source.startswith(to_text(os.path.sep)) or source.startswith(u'~'): 

search.append(unfrackpath(source, follow=False)) 

else: 

# base role/play path + templates/files/vars + relative filename 

search.append(os.path.join(path, dirname, source)) 

basedir = unfrackpath(path, follow=False) 

# not told if role, but detect if it is a role and if so make sure you get correct base path 

233 ↛ 236line 233 didn't jump to line 236, because the condition on line 233 was never false if not is_role: 

is_role = self._is_role(path) 

236 ↛ 237line 236 didn't jump to line 237, because the condition on line 236 was never true if is_role and RE_TASKS.search(path): 

basedir = unfrackpath(os.path.dirname(path), follow=False) 

cur_basedir = self._basedir 

self.set_basedir(basedir) 

# resolved base role/play path + templates/files/vars + relative filename 

search.append(unfrackpath(os.path.join(basedir, dirname, source), follow=False)) 

self.set_basedir(cur_basedir) 

245 ↛ 250line 245 didn't jump to line 250, because the condition on line 245 was never false if is_role and not source.endswith(dirname): 

# look in role's tasks dir w/o dirname 

search.append(unfrackpath(os.path.join(basedir, 'tasks', source), follow=False)) 

# try to create absolute path for loader basedir + templates/files/vars + filename 

search.append(unfrackpath(os.path.join(dirname, source), follow=False)) 

# try to create absolute path for loader basedir 

search.append(unfrackpath(os.path.join(basedir, source), follow=False)) 

# try to create absolute path for dirname + filename 

search.append(self.path_dwim(os.path.join(dirname, source))) 

# try to create absolute path for filename 

search.append(self.path_dwim(source)) 

261 ↛ 265line 261 didn't jump to line 265, because the loop on line 261 didn't complete for candidate in search: 

262 ↛ 261line 262 didn't jump to line 261, because the condition on line 262 was never false if os.path.exists(to_bytes(candidate, errors='surrogate_or_strict')): 

break 

return candidate 

def path_dwim_relative_stack(self, paths, dirname, source, is_role=False): 

''' 

find one file in first path in stack taking roles into account and adding play basedir as fallback 

:arg paths: A list of text strings which are the paths to look for the filename in. 

:arg dirname: A text string representing a directory. The directory 

is prepended to the source to form the path to search for. 

:arg source: A text string which is the filename to search for 

:rtype: A text string 

:returns: An absolute path to the filename ``source`` if found 

:raises: An AnsibleFileNotFound Exception if the file is found to exist in the search paths 

''' 

b_dirname = to_bytes(dirname) 

b_source = to_bytes(source) 

result = None 

search = [] 

284 ↛ 285line 284 didn't jump to line 285, because the condition on line 284 was never true if source is None: 

display.warning('Invalid request to find a file that matches a "null" value') 

286 ↛ 292line 286 didn't jump to line 292, because the condition on line 286 was never false elif source and (source.startswith('~') or source.startswith(os.path.sep)): 

# path is absolute, no relative needed, check existence and return source 

test_path = unfrackpath(b_source, follow=False) 

289 ↛ 321line 289 didn't jump to line 321, because the condition on line 289 was never false if os.path.exists(to_bytes(test_path, errors='surrogate_or_strict')): 

result = test_path 

else: 

display.debug(u'evaluation_path:\n\t%s' % '\n\t'.join(paths)) 

for path in paths: 

upath = unfrackpath(path, follow=False) 

b_upath = to_bytes(upath, errors='surrogate_or_strict') 

b_mydir = os.path.dirname(b_upath) 

# if path is in role and 'tasks' not there already, add it into the search 

if (is_role or self._is_role(path)) and b_mydir.endswith(b'tasks'): 

search.append(os.path.join(os.path.dirname(b_mydir), b_dirname, b_source)) 

search.append(os.path.join(b_mydir, b_source)) 

else: 

# don't add dirname if user already is using it in source 

if b_source.split(b'/')[0] != dirname: 

search.append(os.path.join(b_upath, b_dirname, b_source)) 

search.append(os.path.join(b_upath, b_source)) 

# always append basedir as last resort 

# don't add dirname if user already is using it in source 

if b_source.split(b'/')[0] != dirname: 

search.append(os.path.join(to_bytes(self.get_basedir()), b_dirname, b_source)) 

search.append(os.path.join(to_bytes(self.get_basedir()), b_source)) 

display.debug(u'search_path:\n\t%s' % to_text(b'\n\t'.join(search))) 

for b_candidate in search: 

display.vvvvv(u'looking for "%s" at "%s"' % (source, to_text(b_candidate))) 

if os.path.exists(b_candidate): 

result = to_text(b_candidate) 

break 

321 ↛ 322line 321 didn't jump to line 322, because the condition on line 321 was never true if result is None: 

raise AnsibleFileNotFound(file_name=source, paths=[to_text(p) for p in search]) 

return result 

def _create_content_tempfile(self, content): 

''' Create a tempfile containing defined content ''' 

fd, content_tempfile = tempfile.mkstemp() 

f = os.fdopen(fd, 'wb') 

content = to_bytes(content) 

try: 

f.write(content) 

except Exception as err: 

os.remove(content_tempfile) 

raise Exception(err) 

finally: 

f.close() 

return content_tempfile 

def get_real_file(self, file_path, decrypt=True): 

""" 

If the file is vault encrypted return a path to a temporary decrypted file 

If the file is not encrypted then the path is returned 

Temporary files are cleanup in the destructor 

""" 

if not file_path or not isinstance(file_path, (binary_type, text_type)): 

raise AnsibleParserError("Invalid filename: '%s'" % to_native(file_path)) 

b_file_path = to_bytes(file_path, errors='surrogate_or_strict') 

if not self.path_exists(b_file_path) or not self.is_file(b_file_path): 

raise AnsibleFileNotFound(file_name=file_path) 

real_path = self.path_dwim(file_path) 

try: 

if decrypt: 

with open(to_bytes(real_path), 'rb') as f: 

# Limit how much of the file is read since we do not know 

# whether this is a vault file and therefore it could be very 

# large. 

if is_encrypted_file(f, count=len(b_HEADER)): 

# if the file is encrypted and no password was specified, 

# the decrypt call would throw an error, but we check first 

# since the decrypt function doesn't know the file name 

data = f.read() 

if not self._vault.secrets: 

raise AnsibleParserError("A vault password or secret must be specified to decrypt %s" % to_native(file_path)) 

data = self._vault.decrypt(data, filename=real_path) 

# Make a temp file 

real_path = self._create_content_tempfile(data) 

self._tempfiles.add(real_path) 

return real_path 

except (IOError, OSError) as e: 

raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (to_native(real_path), to_native(e)), orig_exc=e) 

def cleanup_tmp_file(self, file_path): 

""" 

Removes any temporary files created from a previous call to 

get_real_file. file_path must be the path returned from a 

previous call to get_real_file. 

""" 

if file_path in self._tempfiles: 

os.unlink(file_path) 

self._tempfiles.remove(file_path) 

def cleanup_all_tmp_files(self): 

391 ↛ 392line 391 didn't jump to line 392, because the loop on line 391 never started for f in self._tempfiles: 

try: 

self.cleanup_tmp_file(f) 

except Exception as e: 

display.warning("Unable to cleanup temp files: %s" % to_native(e))