Edit File: clpassenger.py
# -*- coding: utf-8 -*- # Copyright © Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2019 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT from __future__ import absolute_import from __future__ import print_function from __future__ import division import fcntl import pwd import syslog from datetime import datetime from future.utils import iteritems from future.moves import configparser as ConfigParser import io import logging import os import re import subprocess from clcommon import clcaptain, utils from clcommon.cpapi import userdomains from clcommon.utils import get_file_system_in_which_file_is_stored_on from clcommon.utils import get_file_lines, write_file_lines from clcommon.utils import mod_makedirs from clquota import QuotaWrapper, NoSuchUserException, InsufficientPrivilegesException, IncorrectLimitFormatException, \ GeneralException, NoSuchPackageException, QuotaDisabledException from lveapi import PyLve, PyLveError from secureio import set_user_perm, set_root_perm from typing import Dict, Union # NOQA from .clselectexcept import ClSelectExcept from .utils import file_readlines, file_write, s_partition from .utils import get_abs_rel, mkdir_p, file_read, file_writelines from .utils import get_using_realpath_keys, realpaths_are_equal # logger for clpassenger module logger = logging.getLogger(__name__) logger.setLevel(logging.ERROR) # disable output of logs to console null_handler = logging.StreamHandler(open('/dev/null', 'w')) logger.addHandler(null_handler) HTACCESS_BEGIN = '# DO NOT REMOVE. CLOUDLINUX PASSENGER CONFIGURATION BEGIN' HTACCESS_END = '# DO NOT REMOVE. CLOUDLINUX PASSENGER CONFIGURATION END' RACK_PATH = 'config.ru' RACK_TEMPLATE = r'''app = proc do |env| message = "It works!\n" version = "Ruby %s\n" % RUBY_VERSION response = [message, version].join("\n") [200, {"Content-Type" => "text/plain"}, [response]] end run app ''' RESTART_PATH = 'tmp/restart.txt' WSGI_PATH = 'passenger_wsgi.py' WSGI_TEMPLATE = r'''import os import sys sys.path.insert(0, os.path.dirname(__file__)) def application(environ, start_response): start_response('200 OK', [('Content-Type', 'text/plain')]) message = 'It works!\n' version = 'Python %s\n' % sys.version.split()[0] response = '\n'.join([message, version]) return [response.encode()] ''' APPJS_PATH = 'app.js' APPJS_TEMPLATE = r'''var http = require('http'); var server = http.createServer(function(req, res) { res.writeHead(200, {'Content-Type': 'text/plain'}); var message = 'It works!\n', version = 'NodeJS ' + process.versions.node + '\n', response = [message, version].join('\n'); res.end(response); }); server.listen(); ''' def drop_root_perm(user): userpwd = pwd.getpwnam(user) set_user_perm(userpwd.pw_uid, userpwd.pw_gid, exit = False) def get_config_lock(config_path, mode): try: conf_file = open(config_path, mode, errors='surrogateescape') fcntl.flock(conf_file.fileno(), fcntl.LOCK_EX) return conf_file except IOError: return None def release_lock(lock_file): try: lock_file.close() except: pass def write_config(user, config_path, config): """ Write config with locking. Drop permissions if method called as root. """ permissions_dropped = False # we must drop & restore permissions only when we # call this method as root, otherwise we can get unexpected # behavior when we drop permission on higher level, # and restore them here, in deep tree of method calls if os.getegid() == 0 or os.geteuid() == 0: drop_root_perm(user) permissions_dropped = True config_file = None try: check_and_createdir(config_path) config_file = get_config_lock(config_path, 'r') file_content = io.StringIO() config.write(file_content) clcaptain.write(config_path, file_content.getvalue()) # clcaptain raises exception from clcommon.utils.ExternalProgramFailes, IOError # exception ClSelectExcept.ExternalProgramFailed has the same name but different except (IOError, OSError, ClSelectExcept.UnableToSaveData, utils.ExternalProgramFailed) as e: syslog.syslog(syslog.LOG_WARNING, "Can't write {}: {}".format(config_path, e)) finally: release_lock(config_file) if permissions_dropped: set_root_perm(exit=False) def check_and_createdir(path): user_backup_path = os.path.dirname(path) if not os.path.isdir(user_backup_path): try: clcaptain.mkdir(user_backup_path) # clcaptain raises exception from clcommon.utils.ExternalProgramFailes # exception ClSelectExcept.ExternalProgramFailed has the same name but different except (OSError, ClSelectExcept.ExternalProgramFailed, utils.ExternalProgramFailed) as e: raise ClSelectExcept.UnableToSaveData(user_backup_path, e) def get_htaccess_cache_path(user): userpwd = pwd.getpwnam(user) return os.path.join(userpwd.pw_dir, '.cl.selector', 'htaccess_cache') def _get_info_about_htaccess_cache_file(path_to_file): # type: (str) -> Dict """ Get info (stat, first n symbols and file system in which file is stored) about htaccess_cache file """ time_format = '%Y-%m-%d %H:%M:%S' number_of_symbols = 100 file_info = {} if os.path.exists(path_to_file): try: file_stat = os.stat(path_to_file) file_info['file_size'] = file_stat.st_size file_info['gid'] = file_stat.st_gid file_info['uid'] = file_stat.st_uid file_info['permissions'] = oct(file_stat.st_mode) file_info['last_access'] = datetime.fromtimestamp(file_stat.st_atime).strftime(time_format) file_info['last_modification'] = datetime.fromtimestamp(file_stat.st_mtime).strftime(time_format) # Not necessary to read file and get file system if it's empty if file_info['file_size'] == 0: return file_info try: with open(path_to_file, 'r') as f: file_info['first_symbols'] = f.read(number_of_symbols) # move to n symbol from end file f.seek(-number_of_symbols, 2) file_info['last_symbols'] = f.read(number_of_symbols) except (OSError, IOError) as err: file_info['error'] = 'We cannot get first and last %s symbols from "%s" file. Exception: %s' % ( number_of_symbols, path_to_file, err, ) file_info['file_system'] = get_file_system_in_which_file_is_stored_on(path_to_file)['details'] except (OSError, IOError) as err: file_info['error'] = 'We cannot get info about "%s" file. Exception: %s' % ( path_to_file, err, ) return file_info def _get_user_lve_limits(user_uid): # type: (int) -> Union[Dict[int, int, int, int, int, int, int], Dict[str]] """ Getting user lve limits for logging those for next debug """ result = dict() try: py_lve = PyLve() py_lve.initialize() user_limits = py_lve.lve_info(user_uid) result['cpu'] = user_limits.ls_cpu / user_limits.ls_cpu_weight result['pmem'] = user_limits.ls_memory_phy result['vmem'] = user_limits.ls_memory result['io'] = user_limits.ls_io result['iops'] = user_limits.ls_iops result['ep'] = user_limits.ls_enters result['nproc'] = user_limits.ls_nproc except PyLveError as err: result['error'] = 'We cannot get lve limits for user with uid "%s". Exception: %s' % ( user_uid, err, ) return result def _get_user_quota_limits(user_uid): # type: (int) -> Union[Dict[str, str, str], Dict[str]] """ Getting user quota limits for logging those for next debug """ result = dict() user_uid = str(user_uid) try: quota_wrapper = QuotaWrapper() user_quotas = quota_wrapper.get_user_limits(user_uid)[user_uid] result = user_quotas except ( NoSuchUserException, NoSuchPackageException, InsufficientPrivilegesException, GeneralException, IncorrectLimitFormatException, QuotaDisabledException, IOError, OSError ) as err: result['error'] = 'We cannot get quota limits for user with uid "%s". Exception: %s' % ( user_uid, err, ) return result def _log_debug_info_about_user_and_config_file(user, config_path, error): # type: (str, str, Exception) -> None """ Logging info (lve & quota limits) about user and info (stat info, first & last n symbols) about config file """ file_info = _get_info_about_htaccess_cache_file(config_path) debug_info = dict() debug_info['config_file_info'] = file_info debug_info['user_info'] = dict() try: user_uid = pwd.getpwnam(user).pw_uid except KeyError as err: debug_info['user_info']['error'] = 'User "%s" does not exists. Exception: %s' % ( user, err, ) user_uid = None if user_uid is not None: debug_info['user_info']['lve_limits'] = dict() debug_info['user_info']['lve_limits'].update(_get_user_lve_limits(user_uid)) debug_info['user_info']['quota_limits'] = dict() debug_info['user_info']['quota_limits'].update(_get_user_quota_limits(user_uid)) logger.exception(error, exc_info=True, extra=debug_info) def read_config(user): config = ConfigParser.RawConfigParser(strict=False) config_path = get_htaccess_cache_path(user) config_file = get_config_lock(config_path, 'r') if config_file is not None: try: config.readfp(config_file) # LU-1035 except (IOError, OSError) as err: # Logging additional information for next debug _log_debug_info_about_user_and_config_file(user, config_path, err) # LU-1032 except (ConfigParser.ParsingError, ConfigParser.MissingSectionHeaderError): _unlink(config_path) syslog.syslog(syslog.LOG_WARNING, "Config {} is broken.".format(config_path)) # if cought ParsingError - return Empty config config = ConfigParser.RawConfigParser(strict=False) finally: release_lock(config_file) return config, config_path def get_htaccess_cache(user, doc_root): config, _ = read_config(user) if config.has_section(doc_root): try: htaccess_list = config.get(doc_root, 'htaccess_list').split(',') return htaccess_list except ConfigParser.NoOptionError: return None return None def write_htaccess_cache(user, doc_root, data): data = data.split('\n') data = list(filter(bool, data)) config, config_path = read_config(user) if not config.has_section(doc_root): config.add_section(doc_root) config.set(doc_root, 'htaccess_list', ','.join(data)) write_config(user, config_path, config) def update_htaccess_cache(user, path_to_file, doc_root): config, config_path = read_config(user) if config.has_section(doc_root): htaccess_list = config.get(doc_root, 'htaccess_list').split(',') else: config.add_section(doc_root) config.set(doc_root, 'htaccess_list', '') htaccess_list = [] if path_to_file not in htaccess_list: htaccess_list.append(path_to_file) htaccess_list = list(filter(bool, htaccess_list)) config.set(doc_root, 'htaccess_list', ','.join(htaccess_list)) write_config(user, config_path, config) def remove_passenger_lines_from_htaccess(htaccess_filename): """ Removes clpassenger lines from .htaccess to stop application :param htaccess_filename: Application .htaccess path :return: None """ lines = file_readlines(htaccess_filename, errors='surrogateescape') new_lines = [] in_config = False for line in lines: if line.startswith(HTACCESS_BEGIN): in_config = True if line.startswith(HTACCESS_END): in_config = False continue if not in_config: new_lines.append(line) # write new .htaccess new_lines = rm_double_empty_lines(new_lines) file_writelines(htaccess_filename, new_lines, 'w', errors='surrogateescape') def configure(user, directory, alias, interpreter, binary, populate=True, action=None, doc_root=None, startup_file=APPJS_PATH, passenger_log_file=None): """ Configure passenger application :param user: name of unix user :param directory: name of dir in user home :param alias: alias of application :param interpreter: interpreter which execute application :param binary: binary of interpreter that execute application :param populate: True if application have to be be populated :param action: action with apllication. can be transit or None :param doc_root: doc_root :param startup_file: start application file :param passenger_log_file: Passenger log filename to write to app's .htaccess :return: None """ abs_dir, _ = get_abs_rel(user, directory) if os.path.exists(abs_dir) and not os.path.isdir(abs_dir): raise ClSelectExcept.WebAppError( 'Destination exists and it is not a directory') if interpreter not in ('python', 'ruby', 'nodejs'): raise ClSelectExcept.InterpreterError( "Unsupported interpreter ('%s')" % interpreter) user_summary = summary(user) try: app_summary = get_using_realpath_keys(user, directory, user_summary) except KeyError: if doc_root is None: raise ClSelectExcept.NoSuchApplication( 'No such application (or application not configured) "%s"' % directory) else: if action != 'transit': exists_dir = app_summary['directory'] raise ClSelectExcept.WebAppError("Specified directory already used by '%s'" % exists_dir) if not doc_root: doc_root = app_summary['docroot'] # Alias, which is empty, means that user passed uri equaled to doc root # and we don't want to normalize the alias, because normalized empty # alias is point and that alias doesn't work in htaccess if alias != '': alias = os.path.normpath(alias) abs_alias, _ = get_abs_rel(user, os.path.join(doc_root, alias)) htaccess = os.path.join(abs_alias, '.htaccess') htaccess_needs_update = True if os.path.exists(htaccess): htaccess_raw = file_read(htaccess, errors='surrogateescape') if HTACCESS_BEGIN in htaccess_raw: for item in user_summary.values(): # The condition allows to detect common part of aliases # For details see commit message item_alias = os.path.normpath(item['alias']) + os.sep if os.path.dirname(os.path.commonprefix([item_alias, alias + os.sep])) != '': exists_dir = item['directory'] if exists_dir != abs_dir: raise ClSelectExcept.WebAppError( "Specified alias is already used by the other " "application: '%s'. Please, specify another application url." % exists_dir) else: # Do not write to .htaccess, it is already correct htaccess_needs_update = False lines = htaccess_raw.splitlines() else: lines = [] if htaccess_needs_update: lines.append('') lines.append(HTACCESS_BEGIN) lines.append('PassengerAppRoot "%s"' % abs_dir) lines.append('PassengerBaseURI "/%s"' % alias) lines.append('Passenger%s "%s"' % (interpreter.title(), binary)) # for some reason autodetect of `app.js` is not working if interpreter == 'nodejs': lines.append('PassengerAppType node') lines.append('PassengerStartupFile %s' % startup_file) # append PassengerAppLogFile directive if need if passenger_log_file and interpreter in ('python', 'nodejs'): lines.append('PassengerAppLogFile "%s"' % passenger_log_file) lines.append(HTACCESS_END) lines = rm_double_empty_lines(lines) mkdir_p(abs_alias) file_writelines(htaccess, ('%s\n' % line for line in lines), errors='surrogateescape') update_htaccess_cache(user, htaccess, doc_root) if populate: # Also creates startup_file populate_app(user, directory, interpreter, startup_file=startup_file) def fix_homedir(user): for domain_alias, data in iteritems(_summary(user)): _, alias = domain_alias old_home = os.path.commonprefix((data['directory'], data['binary'])) _, _, directory = s_partition(data['directory'], old_home) # old python selector has binary as file # and get_abs_rel does realpath() # while new selector binary is symlink # and realpath works wrong binary_dir = os.path.dirname(data['binary']) binary_name = os.path.basename(data['binary']) _, _, _binary = s_partition(binary_dir, old_home) binary = os.path.join(get_abs_rel(user, _binary)[0], binary_name) htaccess_path = data['htaccess'] _unconfigure(htaccess_path) configure(user, directory, alias, data['interpreter'], binary, doc_root=data['docroot']) def move(user, directory, old_alias, new_alias, old_doc_root=None, new_doc_root=None): app_data = get_using_realpath_keys(user, directory, summary(user)) old_doc_root = old_doc_root or app_data['docroot'] new_doc_root = new_doc_root or old_doc_root old_abs_alias = os.path.join(old_doc_root, old_alias) old_htaccess = os.path.join(old_abs_alias, '.htaccess') new_abs_alias = os.path.join(new_doc_root, new_alias) new_htaccess = os.path.join(new_abs_alias, '.htaccess') if not realpaths_are_equal(user, old_htaccess, new_htaccess): _unconfigure(old_htaccess) lines = file_readlines(old_htaccess, errors='surrogateescape') open(old_htaccess, 'w').close() file_writelines(new_htaccess, lines, 'a', errors='surrogateescape') update_htaccess_cache(user, new_htaccess, new_doc_root) def purge(user): for directory in summary(user): unconfigure(user, directory) def populate_app(user, directory, interpreter, startup_file=APPJS_PATH): """ Populate application :param user: name of unix user :param directory: application path in user's home :param interpreter: interpreter which run application :param startup_file: main application file :return: None """ abs_dir, rel_dir = get_abs_rel(user, directory) app_public = os.path.join(abs_dir, 'public') app_tmp = os.path.join(abs_dir, 'tmp') mkdir_p(app_public) mkdir_p(app_tmp) app_configru = os.path.join(abs_dir, RACK_PATH) app_wsgi = os.path.join(abs_dir, WSGI_PATH) app_js = os.path.join(abs_dir, startup_file) configru_installed = os.path.isfile(app_configru) wsgi_installed = os.path.isfile(app_wsgi) appjs_installed = os.path.isfile(app_js) if configru_installed: configru_unchanged = file_read(app_configru) == RACK_TEMPLATE if wsgi_installed: wsgi_unchanged = file_read(app_wsgi) == WSGI_TEMPLATE if appjs_installed: appjs_unchanged = file_read(app_js) == APPJS_TEMPLATE if interpreter == 'python': if not wsgi_installed: file_write(app_wsgi, WSGI_TEMPLATE) if configru_installed and configru_unchanged: _unlink(app_configru) _unlink(app_js) elif interpreter == 'ruby': if not configru_installed: file_write(app_configru, RACK_TEMPLATE, 'w') if wsgi_installed and wsgi_unchanged: _unlink(app_wsgi) _unlink(app_js) elif interpreter == 'nodejs': if not appjs_installed: # add ability to specify startup path # like 'not/existing/subdir/app.js' dir_path = os.path.dirname(app_js) if not os.path.isdir(dir_path): mod_makedirs(dir_path, 0o755) file_write(app_js, APPJS_TEMPLATE) if appjs_installed and appjs_unchanged: _unlink(app_configru) _unlink(app_wsgi) restart(user, directory) def _unlink(path): try: os.unlink(path) except OSError: pass def _summary(user, userdomains_data=None): # TODO PTCLLIB-132 # it's using cache info about users domains # this mechanism should be removed after implementing of caching on DA for userdomains cpapi method domain_docroot_pairs = userdomains(user) if userdomains_data is None else userdomains_data domain_alias_docroot = [] for domain, doc_root in domain_docroot_pairs: if doc_root is None: continue htaccess_cache = get_htaccess_cache(user, doc_root) if not htaccess_cache: p = subprocess.Popen(['/bin/find', doc_root, '-name', '.htaccess'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) (stdoutdata, _) = p.communicate() write_htaccess_cache(user, doc_root, stdoutdata) htaccess_cache = get_htaccess_cache(user, doc_root) if htaccess_cache is None: # if write_htaccess_cache was unsuccessful, we would still get None here continue for ht_path in htaccess_cache: if ht_path: alias = os.path.dirname(ht_path) domain_alias_docroot.append((domain, alias, doc_root)) return _htaccess_summary(user, domain_alias_docroot) def _htaccess_summary(user, domain_alias_docroot): summ = {} for domain, alias, doc_root in domain_alias_docroot: htaccess = os.path.join(alias, '.htaccess') try: htaccess_raw = file_read(htaccess, errors='surrogateescape') except (IOError, OSError): continue approot = re.search( '^PassengerAppRoot\s+"?(?P<directory>.+?)"?$', htaccess_raw, re.MULTILINE) if not approot: continue interpreter = re.search( '^Passenger(?P<interpreter>Python|Ruby|Nodejs)\s+' '"?(?P<binary>.+?)"?$', htaccess_raw, re.MULTILINE) if not interpreter: continue alias_abs, _ = get_abs_rel(user, alias) doc_root_abs, _ = get_abs_rel(user, doc_root) _, _, alias = s_partition(alias_abs, doc_root_abs) alias = alias.lstrip(os.sep) domain_alias = (domain, alias,) # detect what alias belongs domain appuri = re.search('^PassengerBaseURI\s+"?(?P<appuri>.+?)"?$', htaccess_raw, re.MULTILINE) if appuri and not compare_aliases(appuri.groupdict()['appuri'], alias): continue summ[domain_alias] = { 'htaccess': htaccess, 'domain': domain, 'docroot': doc_root, 'directory': approot.groupdict()['directory'], 'interpreter': interpreter.groupdict()['interpreter'].lower(), 'binary': interpreter.groupdict()['binary'], } return summ def compare_aliases(alias1, alias2): return os.path.normpath(alias1.strip('/')) == os.path.normpath(alias2.strip('/')) #FIXME: Need join/rewrite "summary" and "_summary" functions def summary(user, userdomains_data=None): summ_result = {} for domain_alias, value in iteritems(_summary(user, userdomains_data=userdomains_data)): domain, alias = domain_alias app_root = value['directory'] try: _, directory = get_abs_rel(user, app_root) except ClSelectExcept.WrongData: syslog.syslog( syslog.LOG_WARNING, '{} is broken, directory {} is not in user\'s home.'.format( os.path.join(alias, '.htaccess'), app_root )) continue value['alias'] = alias try: app_summary = get_using_realpath_keys(user, directory, summ_result) except KeyError: value['domains'] = [domain] summ_result[directory] = value else: # add domains key if directory has multiple domains if 'domains' not in app_summary: app_summary['domains'] = [] else: app_summary['domains'].append(domain) return summ_result def unconfigure(user, directory): app_data = get_using_realpath_keys(user, directory, summary(user)) htaccess = app_data['htaccess'] _unconfigure(htaccess) def _unconfigure(htaccess): htaccess_raw = file_read(htaccess, errors='surrogateescape') lines = htaccess_raw.splitlines() new_lines = [] in_config = False for line in lines: if line == HTACCESS_BEGIN: in_config = True continue if line == HTACCESS_END: in_config = False continue if in_config: continue new_lines.append(line) lines = rm_double_empty_lines(new_lines) file_writelines(htaccess, ('%s\n' % line for line in lines), 'w', errors='surrogateescape') def iter_path(root, sub): for p in sub.split(os.sep): root = os.path.join(root, p) yield root def restart(user, directory): abs_dir, _ = get_abs_rel(user, directory) if not os.path.exists(abs_dir): raise ClSelectExcept.MissingApprootDirectory("Missing directory %(abs_dir)s" % {'abs_dir': abs_dir}) tmp_dir = os.path.join(abs_dir, 'tmp') if not os.path.exists(tmp_dir): os.mkdir(tmp_dir) app_restart = os.path.join(abs_dir, RESTART_PATH) # imitation system 'touch' if os.path.exists(app_restart): os.utime(app_restart, None) else: open(app_restart, 'a').close() def rm_double_empty_lines(lines): _lines = [] empty_line = True for line in lines: if line.strip(): empty_line = False elif empty_line: continue else: empty_line = True _lines.append(line) if empty_line: return _lines[:-1] return _lines