diff --git a/scripts/crudini b/scripts/crudini index 49971f5..e56e61e 100755 --- a/scripts/crudini +++ b/scripts/crudini @@ -25,21 +25,17 @@ except ModuleNotFoundError: print(f"System hostname: {socket.gethostname()}") exit(991) import os -import shlex +import re import shutil import string import tempfile -from iniparse import ini -orig_ol_init = ini.OptionLine.__init__ -ini.OptionLine.__init__ = lambda self, name, value, separator='=', comment=None, comment_separator=None, comment_offset=-1, line=None: \ - orig_ol_init(self, name, value, separator, comment, comment_separator, comment_offset, line) - - if sys.version_info[0] >= 3: + import shlex as pipes from io import StringIO import configparser else: + import pipes from cStringIO import StringIO import ConfigParser as configparser @@ -60,29 +56,70 @@ def delete_if_exists(path): raise -# TODO: support configurable options for various ini variants. -# For now just support parameters without '=' specified +def file_is_closed(stdfile): + if not stdfile: + # python3 sets sys.stdin etc. to None if closed + return True + else: + # python2 needs to be checked + try: + os.fstat(stdfile.fileno()) + except EnvironmentError as e: + if e.errno == errno.EBADF: + return True + return False + + +# Adjustments to iniparse to optionally use name=value format (nospace) +# and support for parameters without '=' specified class CrudiniInputFilter(): - def __init__(self, fp): + def __init__(self, fp, iniopt): self.fp = fp + self.iniopt = iniopt self.crudini_no_arg = False + self.windows_eol = None + # Note [ \t] used rather than \s to avoid adjusting \r\n when no value + # Unicode spacing around the delimiter would be very unusual anyway + self.delimiter_spacing = re.compile(r'(.*?)[ \t]*([:=])[ \t]*(.*)') def readline(self): line = self.fp.readline() # XXX: This doesn't handle ;inline comments. - # Really should be done within inparse. - if (line and line[0] not in '[ \t#;\n\r' and - '=' not in line and ':' not in line): - self.crudini_no_arg = True - line = line[:-1] + ' = crudini_no_arg\n' + # Really should be done within iniparse. + + # Detect windows format files + # so we can undo iniparse auto conversion to unix + if self.windows_eol is None: + if line: + self.windows_eol = len(line) >= 2 and line[-2] == '\r' + else: + self.windows_eol = os.name == 'nt' + + if line and line[0] not in '[ \t#;\n\r': + + if '=' not in line and ':' not in line: + self.crudini_no_arg = True + line = line.rstrip() + ' = crudini_no_arg\n' + + if 'nospace' in self.iniopt: + # Convert _all_ existing params. New params are handled in + # the iniparse specialization in CrudiniConfigParser() + + # Note if we wanted an option to only convert specified params + # we could do it with special ${value}_crudini_no_space values + # that were then adjusted on output like for crudini_no_arg + # But if need to remove the spacing, then should for all params + + line = self.delimiter_spacing.sub(r'\1\2\3', line) + return line # XXX: should be done in iniparse. Used to # add support for ini files without a section class AddDefaultSection(CrudiniInputFilter): - def __init__(self, fp): - CrudiniInputFilter.__init__(self, fp) + def __init__(self, fp, iniopt): + CrudiniInputFilter.__init__(self, fp, iniopt) self.first = True def readline(self): @@ -104,15 +141,15 @@ class FileLock(object): self.locked = False if os.name == 'nt': - import msvcrt - + # XXX: msvcrt.locking is problematic on windows + # See the history of the portalocker implementation for example: + # https://github.com/WoLpH/portalocker/commits/develop/portalocker + # That would involve needing a new pywin32 dependency, + # so instead we avoid locking on windows for now. def lock(self): - msvcrt.locking(self.fp, msvcrt.LK_LOCK, 1) self.locked = True def unlock(self): - if self.locked: - msvcrt.locking(self.fp, msvcrt.LK_UNLCK, 1) self.locked = False else: @@ -155,18 +192,24 @@ class LockedFile(FileLock): atexit.register(self.delete) open_mode = os.O_RDONLY + open_args = {} if operation != "--get": # We're only reading here, but we check now for write # permissions we'll need in --inplace case to avoid # redundant processing. - # Also an exlusive lock needs write perms anyway. + # Also an exclusive lock needs write perms anyway. open_mode = os.O_RDWR if create and operation != '--del': open_mode += os.O_CREAT + # Don't convert line endings, so we maintain CRLF in files + if sys.version_info[0] >= 3: + open_args = {'newline': ''} + try: - self.fp = os.fdopen(os.open(self.filename, open_mode, 0o666)) + self.fp = os.fdopen(os.open(self.filename, open_mode, 0o666), + **open_args) if inplace: # In general readers (--get) are protected by file_replace(), # but using read lock here gives AC of the ACID properties @@ -177,8 +220,10 @@ class LockedFile(FileLock): # The file may have been renamed since the open so recheck while True: self.lock() - fpnew = os.fdopen(os.open(self.filename, open_mode, 0o666)) - if os.path.sameopenfile(self.fp.fileno(), fpnew.fileno()): + fpnew = os.fdopen(os.open(self.filename, open_mode, 0o666), + **open_args) + if (os.name == 'nt' or + os.path.sameopenfile(self.fp.fileno(), fpnew.fileno())): # Note we don't fpnew.close() here as that would break # any existing fcntl lock (fcntl.lockf is an fcntl lock # despite the name). We don't use flock() at present @@ -214,14 +259,25 @@ class LockedFile(FileLock): # Note we use RawConfigParser rather than SafeConfigParser # to avoid unwanted variable interpolation. # Note iniparse doesn't currently support allow_no_value=True. +# Note iniparse doesn't currently support space_around_delimiters=False. class CrudiniConfigParser(iniparse.RawConfigParser): - def __init__(self, preserve_case=False): + def __init__(self, preserve_case=False, space_around_delimiters=True): iniparse.RawConfigParser.__init__(self) # Without the following we can't have params starting with "rem"! # We ignore lines starting with '%' which mercurial uses to include iniparse.change_comment_syntax('%;#', allow_rem=False) if preserve_case: self.optionxform = str + # Adjust iniparse separator to default to no space around equals + # Note this does NOT convert existing params with spaces, + # that's done in CrudiniInputFilter.readline(). + # XXX: This couples with iniparse internals. + if not space_around_delimiters: + + def new_ol_init(self, name, value, separator="=", *args, **kw): + orig_ol_init(self, name, value, separator, *args, **kw) + orig_ol_init = iniparse.ini.OptionLine.__init__ + iniparse.ini.OptionLine.__init__ = new_ol_init class Print(): @@ -251,13 +307,23 @@ class Print(): class PrintIni(Print): """Use for ini output format.""" + def __init__(self): + self.sep = ' ' + def section_header(self, section): print("[%s]" % section) def name_value(self, name, value, section=None): if value == 'crudini_no_arg': value = '' - print(name, '=', value.replace('\n', '\n ')) + print(name, '=', value.replace('\n', '\n '), sep=self.sep) + + +class PrintIniNoSpace(PrintIni): + """Use for ini output format with no space around equals""" + + def __init__(self): + self.sep = '' class PrintLines(Print): @@ -309,12 +375,12 @@ class PrintSh(Print): sys.exit(1) if value == 'crudini_no_arg': value = '' - sys.stdout.write("%s=%s\n" % (name, shlex.quote(value))) + sys.stdout.write("%s=%s\n" % (name, pipes.quote(value))) class Crudini(): - mode = fmt = update = inplace = cfgfile = output = section = param = \ - value = vlist = listsep = verbose = None + mode = fmt = update = iniopt = inplace = cfgfile = output = section = \ + param = value = vlist = listsep = verbose = None locked_file = None section_explicit_default = False @@ -386,7 +452,7 @@ class Crudini(): os.close(f) if hasattr(os, 'replace'): # >= python 3.3 - os.replace(tmp, name) # atomic even on windos + os.replace(tmp, name) # atomic even on windows elif os.name == 'posix': os.rename(tmp, name) # atomic on POSIX else: @@ -400,10 +466,13 @@ class Crudini(): # rather than continuing to reference the old inode. # This also provides verification in exit status that # this update completes. - O_DIRECTORY = os.O_DIRECTORY if hasattr(os, 'O_DIRECTORY') else 0 - dirfd = os.open(os.path.dirname(name) or '.', O_DIRECTORY) - os.fsync(dirfd) - os.close(dirfd) + if os.name != 'nt': + O_DIRECTORY = 0 + if hasattr(os, 'O_DIRECTORY'): + O_DIRECTORY = os.O_DIRECTORY + dirfd = os.open(os.path.dirname(name) or '.', O_DIRECTORY) + os.fsync(dirfd) + os.close(dirfd) @staticmethod def file_rewrite(name, data): @@ -416,7 +485,11 @@ class Crudini(): - Less Durable as existing data truncated before I/O completes. - Requires write access to file rather than write access to dir. """ - with open(name, 'w') as f: + # Don't convert line endings, so we maintain CRLF in files + if sys.version_info[0] >= 3: + open_args = {'newline': ''} + + with open(name, 'w', **open_args) as f: f.write(data) f.flush() os.fsync(f.fileno()) @@ -433,8 +506,8 @@ class Crudini(): @staticmethod def update_list(curr_val, item, mode, sep): curr_items = [] - use_space = True - if curr_val: + use_space = True # Perhaps have 'nospace' set this default? + if curr_val and curr_val != 'crudini_no_arg': if sep is None: use_space = ' ' in curr_val or ',' not in curr_val curr_items = [v.strip() for v in curr_val.split(",")] @@ -459,22 +532,33 @@ class Crudini(): def usage(self, exitval=0): cmd = os.path.basename(sys.argv[0]) - output = sys.stderr if exitval else sys.stdout + if exitval or file_is_closed(sys.stdout): + output = sys.stderr + else: + output = sys.stdout output.write("""\ A utility for manipulating ini files -Usage: %s --set [OPTION]... config_file section [param] [value] - or: %s --get [OPTION]... config_file [section] [param] - or: %s --del [OPTION]... config_file section [param] [list value] - or: %s --merge [OPTION]... config_file [section] +Usage: + + %s --set [OPTION]... config_file section [param] [value] + %s --get [OPTION]... config_file [section] [param] + %s --del [OPTION]... config_file section [param] [list value] + %s --merge [OPTION]... config_file [section] + + SECTION can be empty ("") or "DEFAULT" in which case, + params not in a section, i.e. global parameters are operated on. + If 'DEFAULT' is used with --set, an explicit [DEFAULT] section is added. Options: --existing[=WHAT] For --set, --del and --merge, fail if item is missing, - where WHAT is 'file', 'section', or 'param', or if - not specified; all specified items. + where WHAT is 'file', 'section', or 'param', + or if WHAT not specified; all specified items. --format=FMT For --get, select the output FMT. - Formats are sh,ini,lines + Formats are 'sh','ini','lines' + --ini-options=OPT Set options for handling ini files. Options are: + 'nospace': use format name=value not name = value --inplace Lock and write files in place. This is not atomic but has less restrictions than the default replacement method. @@ -505,6 +589,7 @@ Options: 'format=', 'get', 'help', + 'ini-options=', 'inplace', 'list', 'list-sep=', @@ -537,6 +622,12 @@ Options: if self.fmt not in ('sh', 'ini', 'lines'): error('--format not recognized: %s' % self.fmt) self.usage(1) + elif o in ('--ini-options',): + self.iniopt = a.split(',') + for opt in self.iniopt: + if opt not in ('nospace'): + error('--ini-options not recognized: %s' % opt) + self.usage(1) elif o in ('--existing',): self.update = a or 'param' # 'param' implies all must exist if self.update not in ('file', 'section', 'param'): @@ -566,6 +657,11 @@ Options: if not self.output: self.output = self.cfgfile + if file_is_closed(sys.stdout) \ + and (self.output == '-' or self.mode == '--get'): + error("stdout is closed") + sys.exit(1) + if self.cfgfile is None: self.usage(1) if self.section is None and self.mode in ('--del', '--set'): @@ -594,12 +690,24 @@ Options: error("param names should not start with '[': %s" % self.param) sys.exit(1) + if not self.iniopt: + self.iniopt = () + # A "param=with=equals = value" line can not be found with --get + # so avoid the ambiguity. Restrict to 'nospace' to allow hack in + # https://github.com/pixelb/crudini/issues/33#issuecomment-1151253988 + if 'nospace' in self.iniopt and self.param and '=' in self.param: + error("param names should not contain '=': %s" % self.param) + sys.exit(1) + if self.fmt == 'lines': self._print = PrintLines() elif self.fmt == 'sh': self._print = PrintSh() elif self.fmt == 'ini': - self._print = PrintIni() + if 'nospace' in self.iniopt: + self._print = PrintIniNoSpace() + else: + self._print = PrintIni() else: self._print = Print() @@ -641,13 +749,16 @@ Options: fp = StringIO(self.data) if add_default: - fp = AddDefaultSection(fp) + fp = AddDefaultSection(fp, self.iniopt) else: - fp = CrudiniInputFilter(fp) + fp = CrudiniInputFilter(fp, self.iniopt) - conf = CrudiniConfigParser(preserve_case=preserve_case) + conf = CrudiniConfigParser(preserve_case=preserve_case, + space_around_delimiters=( + 'nospace' not in self.iniopt)) conf.readfp(fp) self.crudini_no_arg = fp.crudini_no_arg + self.windows_eol = fp.windows_eol return conf except EnvironmentError as e: error(str(e)) @@ -660,6 +771,9 @@ Options: if filename != '-': self.locked_file = LockedFile(filename, self.mode, self.inplace, not self.update) + elif file_is_closed(sys.stdin): + error("stdin is closed") + sys.exit(1) try: conf = self._parse_file(filename, preserve_case=preserve_case) @@ -722,14 +836,30 @@ Options: self.conf.add_section(section) if param is not None: - if self.update not in ('param', 'section'): - try: - curr_val = self.conf.get(section, param) - except configparser.NoOptionError: - if self.mode == "--del": + try: + curr_val = self.conf.get(section, param) + except configparser.NoOptionError: + if self.mode == "--del": + if self.update not in ('param', 'section'): return + if value is None: - value = 'crudini_no_arg' if self.crudini_no_arg else '' + # Unspecified param should clear list. This will also force + # existing param "flags" or new params to use '=' delimiter. + if self.vlist: + curr_val = '' + + if curr_val == 'crudini_no_arg': + # param already exists without delimiter + return + elif curr_val is None and self.crudini_no_arg: + # some params exist without delimiter + # so default new param to not use one + value = 'crudini_no_arg' + else: + # Otherwise use a delimeter + value = '' + if self.vlist: value = self.update_list( curr_val, @@ -854,7 +984,7 @@ Options: self._print.name_value(None, None, section) def run(self): - if sys.stdin.isatty(): + if not file_is_closed(sys.stdin) and sys.stdin.isatty(): sys.excepthook = Crudini.cli_exception Crudini.init_iniparse_defaultsect() @@ -931,19 +1061,28 @@ Options: else: str_data = str_data.replace(default_sect, '', 1) + if self.windows_eol: + # iniparse uses '\n' for new/updated items + # so reset all to windows format + str_data = str_data.replace('\r\n', '\n') + str_data = str_data.replace('\n', '\r\n') + if self.crudini_no_arg: - # This is the main case - str_data = str_data.replace(' = crudini_no_arg', '') - # Handle setting empty values for existing param= format - str_data = str_data.replace('=crudini_no_arg', '=') - # Handle setting empty values for existing colon: format - str_data = str_data.replace(':crudini_no_arg', ':') + spacing = '' if 'nospace' in self.iniopt else ' ' + str_data = str_data.replace('%s=%scrudini_no_arg' % + (spacing, spacing), '') changed = self.chksum != self._chksum(str_data) if self.output == '-': sys.stdout.write(str_data) elif changed: + if os.name == 'nt': + # Close input file as Windows gives access errors on + # open files. For e.g. see caveats noted at: + # https://bugs.python.org/issue46003 + self.locked_file.delete() + if self.inplace: self.file_rewrite(self.output, str_data) else: @@ -952,7 +1091,7 @@ Options: if self.verbose: def quote_val(val): - return shlex.quote(val).replace('\n', '\\n') + return pipes.quote(val).replace('\n', '\\n') what = ' '.join(map(quote_val, list(filter(bool, [self.mode, self.cfgfile, @@ -963,7 +1102,8 @@ Options: # Finish writing now to consistently handle errors here # (and while excepthook is set) - sys.stdout.flush() + if not file_is_closed(sys.stdout): + sys.stdout.flush() except configparser.ParsingError as e: error('Error parsing %s: %s' % (self.cfgfile, e.message)) sys.exit(1) @@ -980,8 +1120,9 @@ Options: sys.exit(1) # Python3 fix for exception on exit: # https://docs.python.org/3/library/signal.html#note-on-sigpipe - nullf = os.open(os.devnull, os.O_WRONLY) - os.dup2(nullf, sys.stdout.fileno()) + if not file_is_closed(sys.stdout): + nullf = os.open(os.devnull, os.O_WRONLY) + os.dup2(nullf, sys.stdout.fileno()) def main(): @@ -991,4 +1132,3 @@ def main(): if __name__ == "__main__": sys.exit(main()) -