# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# Copyright 2014 SoftLayer Technologies, Inc.
# Copyright 2015 Mirantis, Inc
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
System-level utilities and helper functions.
"""
import errno
import ipaddress
try:
from eventlet import sleep
except ImportError:
from time import sleep
from eventlet.green import socket
import functools
import os
import re
import urllib
import glance_store
from oslo_config import cfg
from oslo_log import log as logging
from oslo_utils import excutils
from oslo_utils import netutils
from oslo_utils import strutils
from webob import exc
from glance.common import exception
from glance.common import location_strategy
from glance.common import timeutils
from glance.common import wsgi
from glance.i18n import _, _LE, _LW
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
# Whitelist of v1 API headers of form x-image-meta-xxx
IMAGE_META_HEADERS = ['x-image-meta-location', 'x-image-meta-size',
'x-image-meta-is_public', 'x-image-meta-disk_format',
'x-image-meta-container_format', 'x-image-meta-name',
'x-image-meta-status', 'x-image-meta-copy_from',
'x-image-meta-uri', 'x-image-meta-checksum',
'x-image-meta-created_at', 'x-image-meta-updated_at',
'x-image-meta-deleted_at', 'x-image-meta-min_ram',
'x-image-meta-min_disk', 'x-image-meta-owner',
'x-image-meta-store', 'x-image-meta-id',
'x-image-meta-protected', 'x-image-meta-deleted',
'x-image-meta-virtual_size']
GLANCE_TEST_SOCKET_FD_STR = 'GLANCE_TEST_SOCKET_FD'
[docs]def chunkreadable(iter, chunk_size=65536):
"""
Wrap a readable iterator with a reader yielding chunks of
a preferred size, otherwise leave iterator unchanged.
:param iter: an iter which may also be readable
:param chunk_size: maximum size of chunk
"""
return chunkiter(iter, chunk_size) if hasattr(iter, 'read') else iter
[docs]def chunkiter(fp, chunk_size=65536):
"""
Return an iterator to a file-like obj which yields fixed size chunks
:param fp: a file-like object
:param chunk_size: maximum size of chunk
"""
while True:
chunk = fp.read(chunk_size)
if chunk:
yield chunk
else:
break
[docs]def cooperative_iter(iter):
"""
Return an iterator which schedules after each
iteration. This can prevent eventlet thread starvation.
:param iter: an iterator to wrap
"""
try:
for chunk in iter:
sleep(0)
yield chunk
except Exception as err:
with excutils.save_and_reraise_exception():
msg = _LE("Error: cooperative_iter exception %s") % err
LOG.error(msg)
[docs]def cooperative_read(fd):
"""
Wrap a file descriptor's read with a partial function which schedules
after each read. This can prevent eventlet thread starvation.
:param fd: a file descriptor to wrap
"""
def readfn(*args):
result = fd.read(*args)
sleep(0)
return result
return readfn
MAX_COOP_READER_BUFFER_SIZE = 134217728 # 128M seems like a sane buffer limit
CONF.import_group('import_filtering_opts',
'glance.async_.flows._internal_plugins')
[docs]def normalize_hostname(host):
"""Normalize IP address to standard format or return hostname.
Uses ipaddress module to validate and normalize IP addresses, rejecting
encoded formats. For hostnames, requires DNS resolution to ensure they
are valid and not encoded IP attempts.
:param host: hostname or IP address
:returns: normalized IP address, hostname unchanged, or None
"""
if not host:
return host
# NOTE(abhishekk): Try to parse as IPv4. ipaddress module only accepts
# standard format like 127.0.0.1. It rejects encoded formats like
# decimal (2130706433), hex (0x7f000001), or octal (017700000001).
try:
return str(ipaddress.IPv4Address(host))
except ValueError:
pass
# NOTE(abhishekk): Try to parse as IPv6. ipaddress module only accepts
# standard IPv6 format and rejects encoded formats.
try:
return str(ipaddress.IPv6Address(host))
except ValueError:
pass
# NOTE(abhishekk): Not valid IP address, check as hostname. Reject pure
# numeric strings like "2130706433" (decimal encoded IP). ipaddress module
# rejected it, but OS might still resolve using inet_aton() if not blocked.
if host.isdigit():
return None
# NOTE(abhishekk): Reject all numeric strings with dots like "127.1" or
# "10.1". These are shorthand IP addresses. ipaddress module rejects them
# because they need 4 octets, but OS may still resolve them. We block to
# prevent SSRF bypass attacks.
if all(c.isdigit() or c == '.' for c in host):
return None
# NOTE(abhishekk): Add trailing dot to force DNS lookup instead of numeric
# parsing. This blocks encoded IP formats like 0x7f000001 or 127.0x0.0.1
# because they fail DNS lookup. Only real hostnames that resolve via DNS
# are allowed.
testhost = host
if not testhost.endswith('.'):
testhost += '.'
try:
socket.getaddrinfo(testhost, 80)
except socket.gaierror:
# NOTE(abhishekk): DNS resolution failed, reject the hostname
return None
# NOTE(abhishekk): Valid and resolvable hostname, return unchanged
return host
[docs]def default_port(scheme):
"""Return the default port for an external HTTP(S) URI scheme."""
return 443 if scheme == 'https' else 80
[docs]def resolve_pinned_address(hostname, port):
"""Resolve hostname and return the first allowed IP address.
Blocks loopback and link-local addresses (including cloud metadata
endpoints such as 169.254.169.254) unless the host is listed in
allowed_hosts. Private RFC1918 ranges are not blocked so private
clouds can fetch from internal hosts. IPv4-mapped IPv6 addresses
are checked against the embedded IPv4 address. Matching
disallowed_hosts IP entries also reject resolved addresses.
Used for web-download import and other external HTTP(S) fetches that
apply ``import_filtering_opts`` (for example HTTP image locations).
:param hostname: hostname or IP from the external URI
:param port: destination port from the external URI
:returns: IP address string to connect to
:raises ValueError: if the host cannot be used for an external fetch
"""
normalized_host = normalize_hostname(hostname)
if not normalized_host:
raise ValueError('invalid external host: %s' % hostname)
bl_hosts = list(CONF.import_filtering_opts.disallowed_hosts)
wl_hosts = CONF.import_filtering_opts.allowed_hosts
if wl_hosts and bl_hosts:
bl_hosts = []
host_is_whitelisted = bool(wl_hosts and normalized_host in wl_hosts)
try:
try:
addresses = [ipaddress.ip_address(normalized_host)]
except ValueError:
testhost = (normalized_host if normalized_host.endswith('.')
else normalized_host + '.')
addresses = []
for result in socket.getaddrinfo(testhost, port):
ip = result[4][0]
if not ip:
raise ValueError('empty address in getaddrinfo result')
if '%' in ip:
ip = ip.split('%', 1)[0]
addr = ipaddress.ip_address(ip)
if addr not in addresses:
addresses.append(addr)
except (socket.gaierror, ValueError) as exc:
raise ValueError('failed to resolve external host %s: %s' %
(hostname, exc))
for addr in addresses:
if host_is_whitelisted:
return str(addr)
check = addr
if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped:
check = addr.ipv4_mapped
if check.is_loopback or check.is_link_local:
continue
addr_str = str(addr)
blocked = False
for blocked_host in bl_hosts:
if blocked_host == normalized_host:
continue
try:
if addr == ipaddress.ip_address(blocked_host):
blocked = True
break
except ValueError:
if addr_str == blocked_host:
blocked = True
break
if not blocked:
return str(addr)
raise ValueError('no allowed addresses for external host: %s' % hostname)
[docs]def get_validated_address(uri):
"""Validate an external URI and return a pinned destination IP.
Applies ``import_filtering_opts`` scheme/host/port filtering, resolves
DNS, and returns the first allowed address. Raises ValueError if the
URI must not be fetched.
"""
if not uri:
raise ValueError('empty external URI')
parsed_uri = urllib.parse.urlparse(uri)
scheme = parsed_uri.scheme
host = parsed_uri.hostname
port = parsed_uri.port
wl_schemes = CONF.import_filtering_opts.allowed_schemes
bl_schemes = CONF.import_filtering_opts.disallowed_schemes
wl_hosts = CONF.import_filtering_opts.allowed_hosts
bl_hosts = CONF.import_filtering_opts.disallowed_hosts
wl_ports = CONF.import_filtering_opts.allowed_ports
bl_ports = CONF.import_filtering_opts.disallowed_ports
# NOTE(jokke): Checking if both allowed and disallowed are defined and
# logging it to inform only allowed will be obeyed.
if wl_schemes and bl_schemes:
bl_schemes = []
LOG.debug("Both allowed and disallowed schemes has been configured. "
"Will only process allowed list.")
if wl_hosts and bl_hosts:
bl_hosts = []
LOG.debug("Both allowed and disallowed hosts has been configured. "
"Will only process allowed list.")
if wl_ports and bl_ports:
bl_ports = []
LOG.debug("Both allowed and disallowed ports has been configured. "
"Will only process allowed list.")
if not scheme or ((wl_schemes and scheme not in wl_schemes) or
parsed_uri.scheme in bl_schemes):
raise ValueError('scheme not allowed for external URI: %s' % uri)
normalized_host = normalize_hostname(host)
if not normalized_host or (
(wl_hosts and normalized_host not in wl_hosts) or
normalized_host in bl_hosts):
raise ValueError('host not allowed for external URI: %s' % uri)
if port and ((wl_ports and port not in wl_ports) or
port in bl_ports):
raise ValueError('port not allowed for external URI: %s' % uri)
resolve_port = port or default_port(scheme)
return resolve_pinned_address(normalized_host, resolve_port)
[docs]def validate_uri(uri):
"""Return True if the URI passes ``import_filtering_opts`` filtering."""
try:
get_validated_address(uri)
return True
except ValueError:
return False
[docs]class CooperativeReader(object):
"""
An eventlet thread friendly class for reading in image data.
When accessing data either through the iterator or the read method
we perform a sleep to allow a co-operative yield. When there is more than
one image being uploaded/downloaded this prevents eventlet thread
starvation, ie allows all threads to be scheduled periodically rather than
having the same thread be continuously active.
"""
def __init__(self, fd):
"""
:param fd: Underlying image file object
"""
self.fd = fd
self.iterator = None
# NOTE(markwash): if the underlying supports read(), overwrite the
# default iterator-based implementation with cooperative_read which
# is more straightforward
if hasattr(fd, 'read'):
self.read = cooperative_read(fd)
else:
self.iterator = None
self.buffer = b''
self.position = 0
[docs] def read(self, length=None):
"""Return the requested amount of bytes, fetching the next chunk of
the underlying iterator when needed.
This is replaced with cooperative_read in __init__ if the underlying
fd already supports read().
"""
if length is None:
if len(self.buffer) - self.position > 0:
# if no length specified but some data exists in buffer,
# return that data and clear the buffer
result = self.buffer[self.position:]
self.buffer = b''
self.position = 0
return bytes(result)
else:
# otherwise read the next chunk from the underlying iterator
# and return it as a whole. Reset the buffer, as subsequent
# calls may specify the length
try:
if self.iterator is None:
self.iterator = self.__iter__()
return next(self.iterator)
except StopIteration:
return b''
finally:
self.buffer = b''
self.position = 0
else:
result = bytearray()
while len(result) < length:
if self.position < len(self.buffer):
to_read = length - len(result)
chunk = self.buffer[self.position:self.position + to_read]
result.extend(chunk)
# This check is here to prevent potential OOM issues if
# this code is called with unreasonably high values of read
# size. Currently it is only called from the HTTP clients
# of Glance backend stores, which use httplib for data
# streaming, which has readsize hardcoded to 8K, so this
# check should never fire. Regardless it still worths to
# make the check, as the code may be reused somewhere else.
if len(result) >= MAX_COOP_READER_BUFFER_SIZE:
raise exception.LimitExceeded()
self.position += len(chunk)
else:
try:
if self.iterator is None:
self.iterator = self.__iter__()
self.buffer = next(self.iterator)
self.position = 0
except StopIteration:
self.buffer = b''
self.position = 0
return bytes(result)
return bytes(result)
def __iter__(self):
return cooperative_iter(self.fd.__iter__())
[docs]class LimitingReader(object):
"""
Reader designed to fail when reading image data past the configured
allowable amount.
"""
def __init__(self, data, limit,
exception_class=exception.ImageSizeLimitExceeded):
"""
:param data: Underlying image data object
:param limit: maximum number of bytes the reader should allow
:param exception_class: Type of exception to be raised
"""
self.data = data
self.limit = limit
self.bytes_read = 0
self.exception_class = exception_class
def __iter__(self):
for chunk in self.data:
self.bytes_read += len(chunk)
if self.bytes_read > self.limit:
raise self.exception_class()
else:
yield chunk
[docs] def read(self, i):
result = self.data.read(i)
self.bytes_read += len(result)
if self.bytes_read > self.limit:
raise self.exception_class()
return result
[docs]def create_mashup_dict(image_meta):
"""
Returns a dictionary-like mashup of the image core properties
and the image custom properties from given image metadata.
:param image_meta: metadata of image with core and custom properties
"""
d = {}
for key, value in image_meta.items():
if isinstance(value, dict):
for subkey, subvalue in create_mashup_dict(value).items():
if subkey not in image_meta:
d[subkey] = subvalue
else:
d[key] = value
return d
[docs]def safe_mkdirs(path):
try:
os.makedirs(path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
[docs]def mutating(func):
"""Decorator to enforce read-only logic"""
@functools.wraps(func)
def wrapped(self, req, *args, **kwargs):
if req.context.read_only:
msg = "Read-only access"
LOG.debug(msg)
raise exc.HTTPForbidden(msg, request=req,
content_type="text/plain")
return func(self, req, *args, **kwargs)
return wrapped
[docs]def setup_remote_pydev_debug(host, port):
error_msg = _LE('Error setting up the debug environment. Verify that the'
' option pydev_worker_debug_host is pointing to a valid '
'hostname or IP on which a pydev server is listening on'
' the port indicated by pydev_worker_debug_port.')
try:
try:
from pydev import pydevd
except ImportError:
import pydevd
pydevd.settrace(host,
port=port,
stdoutToServer=True,
stderrToServer=True)
return True
except Exception:
with excutils.save_and_reraise_exception():
LOG.exception(error_msg)
[docs]def get_test_suite_socket():
global GLANCE_TEST_SOCKET_FD_STR
if GLANCE_TEST_SOCKET_FD_STR in os.environ:
fd = int(os.environ[GLANCE_TEST_SOCKET_FD_STR])
sock = socket.fromfd(fd, socket.AF_INET, socket.SOCK_STREAM)
sock.listen(CONF.backlog)
del os.environ[GLANCE_TEST_SOCKET_FD_STR]
os.close(fd)
return sock
return None
[docs]def is_valid_hostname(hostname):
"""Verify whether a hostname (not an FQDN) is valid."""
return re.match('^[a-zA-Z0-9-]+$', hostname) is not None
[docs]def is_valid_fqdn(fqdn):
"""Verify whether a host is a valid FQDN."""
return re.match(r'^[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$', fqdn) is not None
[docs]def parse_valid_host_port(host_port):
"""
Given a "host:port" string, attempts to parse it as intelligently as
possible to determine if it is valid. This includes IPv6 [host]:port form,
IPv4 ip:port form, and hostname:port or fqdn:port form.
Invalid inputs will raise a ValueError, while valid inputs will return
a (host, port) tuple where the port will always be of type int.
"""
try:
try:
host, port = netutils.parse_host_port(host_port)
except Exception:
raise ValueError(_('Host and port "%s" is not valid.') % host_port)
if not netutils.is_valid_port(port):
raise ValueError(_('Port "%s" is not valid.') % port)
# First check for valid IPv6 and IPv4 addresses, then a generic
# hostname. Failing those, if the host includes a period, then this
# should pass a very generic FQDN check. The FQDN check for letters at
# the tail end will weed out any hilariously absurd IPv4 addresses.
if not (netutils.is_valid_ipv6(host) or netutils.is_valid_ipv4(host) or
is_valid_hostname(host) or is_valid_fqdn(host)):
raise ValueError(_('Host "%s" is not valid.') % host)
except Exception as ex:
raise ValueError(_('%s '
'Please specify a host:port pair, where host is an '
'IPv4 address, IPv6 address, hostname, or FQDN. If '
'using an IPv6 address, enclose it in brackets '
'separately from the port (i.e., '
'"[fe80::a:b:c]:9876").') % ex)
return (host, int(port))
try:
REGEX_4BYTE_UNICODE = re.compile('[\U00010000-\U0010ffff]')
except re.error:
# UCS-2 build case
REGEX_4BYTE_UNICODE = re.compile('[\uD800-\uDBFF][\uDC00-\uDFFF]')
[docs]def no_4byte_params(f):
"""
Checks that no 4 byte unicode characters are allowed
in dicts' keys/values and string's parameters
"""
def wrapper(*args, **kwargs):
def _is_match(some_str):
return (
isinstance(some_str, str) and
REGEX_4BYTE_UNICODE.findall(some_str) != []
)
def _check_dict(data_dict):
# a dict of dicts has to be checked recursively
for key, value in data_dict.items():
if isinstance(value, dict):
_check_dict(value)
else:
if _is_match(key):
msg = _("Property names can't contain 4 byte unicode.")
raise exception.Invalid(msg)
if _is_match(value):
msg = (_("%s can't contain 4 byte unicode characters.")
% key.title())
raise exception.Invalid(msg)
for data_dict in [arg for arg in args if isinstance(arg, dict)]:
_check_dict(data_dict)
# now check args for str values
for arg in args:
if _is_match(arg):
msg = _("Param values can't contain 4 byte unicode.")
raise exception.Invalid(msg)
# check kwargs as well, as params are passed as kwargs via
# registry calls
_check_dict(kwargs)
return f(*args, **kwargs)
return wrapper
[docs]def stash_conf_values():
"""
Make a copy of some of the current global CONF's settings.
Allows determining if any of these values have changed
when the config is reloaded.
"""
conf = {
'bind_host': CONF.bind_host,
'bind_port': CONF.bind_port,
'backlog': CONF.backlog,
}
return conf
[docs]def split_filter_op(expression):
"""Split operator from threshold in an expression.
Designed for use on a comparative-filtering query field.
When no operator is found, default to an equality comparison.
:param expression: the expression to parse
:returns: a tuple (operator, threshold) parsed from expression
"""
left, sep, right = expression.partition(':')
if sep:
# If the expression is a date of the format ISO 8601 like
# CCYY-MM-DDThh:mm:ss+hh:mm and has no operator, it should
# not be partitioned, and a default operator of eq should be
# assumed.
try:
timeutils.parse_isotime(expression)
op = 'eq'
threshold = expression
except ValueError:
op = left
threshold = right
else:
op = 'eq' # default operator
threshold = left
# NOTE stevelle decoding escaped values may be needed later
return op, threshold
[docs]def validate_quotes(value):
"""Validate filter values
Validation opening/closing quotes in the expression.
"""
open_quotes = True
for i in range(len(value)):
if value[i] == '"':
if i and value[i - 1] == '\\':
continue
if open_quotes:
if i and value[i - 1] != ',':
msg = _("Invalid filter value %s. There is no comma "
"before opening quotation mark.") % value
raise exception.InvalidParameterValue(message=msg)
else:
if i + 1 != len(value) and value[i + 1] != ",":
msg = _("Invalid filter value %s. There is no comma "
"after closing quotation mark.") % value
raise exception.InvalidParameterValue(message=msg)
open_quotes = not open_quotes
if not open_quotes:
msg = _("Invalid filter value %s. The quote is not closed.") % value
raise exception.InvalidParameterValue(message=msg)
[docs]def split_filter_value_for_quotes(value):
"""Split filter values
Split values by commas and quotes for 'in' operator, according api-wg.
"""
validate_quotes(value)
tmp = re.compile(r'''
"( # if found a double-quote
[^\"\\]* # take characters either non-quotes or backslashes
(?:\\. # take backslashes and character after it
[^\"\\]*)* # take characters either non-quotes or backslashes
) # before double-quote
",? # a double-quote with comma maybe
| ([^,]+),? # if not found double-quote take any non-comma
# characters with comma maybe
| , # if we have only comma take empty string
''', re.VERBOSE)
return [val[0] or val[1] for val in re.findall(tmp, value)]
[docs]def evaluate_filter_op(value, operator, threshold):
"""Evaluate a comparison operator.
Designed for use on a comparative-filtering query field.
:param value: evaluated against the operator, as left side of expression
:param operator: any supported filter operation
:param threshold: to compare value against, as right side of expression
:raises InvalidFilterOperatorValue: if an unknown operator is provided
:returns: boolean result of applied comparison
"""
if operator == 'gt':
return value > threshold
elif operator == 'gte':
return value >= threshold
elif operator == 'lt':
return value < threshold
elif operator == 'lte':
return value <= threshold
elif operator == 'neq':
return value != threshold
elif operator == 'eq':
return value == threshold
msg = _("Unable to filter on a unknown operator.")
raise exception.InvalidFilterOperatorValue(msg)
def _get_available_stores():
available_stores = CONF.enabled_backends
stores = []
# Remove reserved stores from the available stores list
for store in available_stores:
# NOTE (abhishekk): http store is readonly and should be
# excluded from the list.
if available_stores[store] == 'http':
continue
if store not in wsgi.RESERVED_STORES:
stores.append(store)
return stores
[docs]def get_stores_from_request(req, body):
"""Processes a supplied request and extract stores from it
:param req: request to process
:param body: request body
:raises glance_store.UnknownScheme: if a store is not valid
:return: a list of stores
"""
if body.get('all_stores', False):
if 'stores' in body or 'x-image-meta-store' in req.headers:
msg = _("All_stores parameter can't be used with "
"x-image-meta-store header or stores parameter")
raise exc.HTTPBadRequest(explanation=msg)
stores = _get_available_stores()
else:
try:
stores = body['stores']
except KeyError:
stores = [req.headers.get('x-image-meta-store',
CONF.glance_store.default_backend)]
else:
if 'x-image-meta-store' in req.headers:
msg = _("Stores parameter and x-image-meta-store header can't "
"be both specified")
raise exc.HTTPBadRequest(explanation=msg)
# Validate each store
for store in stores:
glance_store.get_store_from_store_identifier(store)
return stores
[docs]def sort_image_locations(locations):
if not CONF.enabled_backends:
return location_strategy.get_ordered_locations(locations)
def get_store_weight(location):
store_id = location['metadata'].get('store')
if not store_id:
return 0
try:
store = glance_store.get_store_from_store_identifier(store_id)
except glance_store.exceptions.UnknownScheme:
msg = (_LW("Unable to find store '%s', returning "
"default weight '0'") % store_id)
LOG.warning(msg)
return 0
return store.weight if store is not None else 0
sorted_locations = sorted(locations, key=get_store_weight, reverse=True)
scrubbed = []
for loc in sorted_locations:
scrubbed.append({'store_name': loc["metadata"].get("store")})
LOG.debug(('Sorted locations: %s'), scrubbed)
return sorted_locations
