# keep-py-file: True
#########################################################################
""" updateutils.py  -- Utilities for updating Wingware software

Copyright (c) 1999-2018, Archaeopteryx Software, Inc.  All rights reserved.

Written by Stephan R.A. Deibel and John P. Ehresman

"""

# This is used during bootstrap and should not load any other modules in wingbase

import os
import time
import collections
import tempfile
import shutil
import stat
import sys
import tarfile
import fnmatch
import traceback

# This import is at the top level to avoid imports during computation of the hash,
# because the debugger uses this code to compute hashes identifying imported files
try:
  import hashlib
  makehash = hashlib.sha1
except:
  #print("Failed to import hashlib - debugger may fail to identify some files:")
  #print(traceback.format_exc())
  makehash = None

try:
  import gettext
  if sys.version_info < (3, 0):
    _ = gettext.translation('src_wingbase', fallback=True).ugettext
  else:
    _ = gettext.translation('src_wingbase', fallback=True).gettext
except:
  def _(a):
    return a
  
#----------------------------------------------------------------------
def GetSHA1Digest(filenames, normalize_eol=False):
  """Get SHA1 digest for given files.  If multiple files are given, a single
  digest is computed for all of them appended in alphabetical order."""

  all_txt = []
  for filename in sorted(filenames):
    f = open(filename, 'rb')
    txt = f.read()
    f.close()

    if normalize_eol:
      txt = txt.replace(b'\r\n', b'\n').replace(b'\r', b'\n')
      
    all_txt.append(txt)

  digest = ComputeSHA1Digest(b''.join(all_txt))
  
  return digest

#----------------------------------------------------------------------
def ReadSHA1SourceDigest(filename):
  """Read the source digest for given file.  It is stored in a filenames
  named xxx.shash for a filename of xxx.*"""
  
  sha_file = filename[:filename.rfind('.')] + '.shash'
  if not os.path.isfile(sha_file):
    return ''
  
  f = open(sha_file, 'rb')
  txt = f.read()
  f.close()
  
  if sys.version_info >= (3, 0):
    txt = txt.decode('utf-8')
  
  digest = txt[:txt.find(':')]
  return digest
  
#----------------------------------------------------------------------
def ComputeSHA1Digest(txt):
  """Compute SHA1 digest for given text.  Returns None if the hashlib and sha
  modules are both not available."""

  if makehash is None:
    return None
  
  if not isinstance(txt, bytes):
    txt = txt.encode('utf-8')
  
  s = makehash()
  s.update(txt)
  digest = s.hexdigest()
  
  return digest

#-----------------------------------------------------------------------
def BuildDirectoryCatalog(dirname, omit_names=set(), omit_exts=set()):
  """Build catalog of all files in install_root so we know which files are
  for which OS and have an SHA1 hash of every file.  The partial paths
  in the catalog always use / as separator."""

  def partial_path(root, fn):
    fp = os.path.normpath(os.path.join(root, fn))
    return fp[len(dirname)+1:].replace(os.sep, '/')

  catalog = {}
  for root, subdirs, files in os.walk(dirname):
    for dn in subdirs:
      catalog[partial_path(root, dn)] = ('D', None, None)
    for fn in files:
      fp = os.path.join(root, fn)
      if fn in omit_names:
        continue
      if any(fn.endswith(ext) for ext in omit_exts):
        continue
      digest = GetSHA1Digest([fp], normalize_eol=True)
      source_digest = ReadSHA1SourceDigest(fp)
      catalog[partial_path(root, fn)] = ('F', digest, source_digest)

  return catalog

#-----------------------------------------------------------------------
def WriteDirectoryCatalog(catalog, catfile):
  """Convert given catalog to text and write to given file"""
  
  txt = []
  for fp in catalog:
    ftype, digest, source_digest = catalog[fp]
    if ftype == 'F':
      txt.append('%s|%s|%s|%s' % (ftype, digest, source_digest, fp))
    else:
      txt.append('%s|%s' % (ftype, fp))

  content = '\n'.join(txt) + '\n'
  if sys.version_info < (3, 0):
    content_bytes = content
  else:
    content_bytes = content.encode('utf-8')
    
  f = open(catfile, mode='wb')
  f.write(content_bytes)
  f.close()

#-----------------------------------------------------------------------
def ReadDirectoryCatalog(fn):
  """Read given directory catalog from disk and parse it"""
  
  f = open(fn, 'rb')
  content_bytes = f.read()
  f.close()
  
  if sys.version_info >= (3, 0):
    txt = content_bytes.decode('utf-8')
  else:
    txt = content_bytes

  return ParseDirectoryCatalog(txt)

#----------------------------------------------------------------------
def ParseDirectoryCatalog(txt):
  """Parse the given directory catalog"""
  
  lines = txt.splitlines()
  
  catalog = {}
  for line in lines:
    line = line.strip()
    if not '|'in line:
      return _("Corrupt catalog: Missing | in line: ") + line, None
    ltype = line[:line.find('|')]
    line = line[2:]
    if ltype == 'F':
      digest = line[:line.find('|')]
      line = line[line.find('|')+1:]
      source_digest = line[:line.find('|')]
      line = line[line.find('|')+1:]
      catalog[line] = (ltype, digest, source_digest)
    elif ltype == 'D':
      catalog[line] = (ltype, None, None)
    else:
      return _("Corrupt catalog: Invalid line type: ") + line, None

  return None, catalog

#----------------------------------------------------------------------
def CompareDirectoryCatalogs(install_root, old, new, osname, whole_dirs=[], omitted={},
                             compute_deleted=True):
  """Find changed and deleted files in the update from old to new catalog.
  Set whole_dirs to a list of prefixes for which any file change will include
  that whole directory.  Set omitted to a dict from directory name to
  list of file name patterns to omit.  Set compute_deleted to False to
  prevent computing the set of files that were deleted and removing those
  from the new catalog.  This is done when staging because we're rebuilding
  a catalog from scratch and merging the OSes.  It should never be set to False
  when computing an update from completed install roots since then the
  catalog is also complete and contains all OSes."""
  
  def whole_dir_item(fp):
    for wp in whole_dirs:
      if fp.startswith(wp):
        while wp.endswith('/'):
          wp = wp[:-1]
        return wp
    return None
  
  omitted_but_changed = set()
  def ok_to_include(fp):
    fp = fp.replace('/', os.sep)
    for rdir, omit_list in omitted.items():
      rdir = rdir.replace('/', os.sep)
      if rdir == os.sep:
        dir_matches = (os.sep not in fp)
      else:
        dir_matches = fp.startswith(rdir)
      if dir_matches:
        for omit in omit_list:
          if fnmatch.fnmatch(fp, omit):
            omitted_but_changed.add(fp)
            return False
    return True
    
  # Compute things that were added or changed since the old version
  changed_files = set()               # Files/dirs that should be in an update
  changed_files_in_catalog = set()    # Files that need to be updated in the catalog
  delete_first = set()
  added_whole_dirs = set()
  for fp in new:
    ftype, digest, source_digest= new[fp]
    old_ftype, old_digest, old_source_digest = old.get(fp, (None, None, None))

    # Need to delete anything that changes file type
    if old_ftype is not None and old_ftype != ftype:
        delete_first.add(fp)
        
    # Include files with changed digest or directories not in the old version
    # (to handle the case of a new empty dir); we compare the source digest,
    # if there is one, since digest of the file itself often changes even if
    # the source does not.
    if source_digest:
      changed = source_digest != old_source_digest
    else:
      changed = digest != old_digest
    if changed or fp not in old:
      
      # Include all of some dirs if any file in it changed; we also keep track of files
      # individually so we can generated the new catalog w/ updated entries for those.
      wp = whole_dir_item(fp)
      if wp is not None:
        if wp not in added_whole_dirs:
          changed_files.add(wp)
          delete_first.add(wp)
          added_whole_dirs.add(wp)
        changed_files_in_catalog.add(fp)

      # Other things can be patched file by file
      elif ok_to_include(fp):
        changed_files.add(fp)
        changed_files_in_catalog.add(fp)
  
  # Compute things that were deleted since the old version unless disabled by caller; note
  # that 'new' contains just items for the OS we're building or updating for.
  if compute_deleted:
    deleted = set(old)
    deleted.difference_update(new)
    for ditem in old:

      if ditem not in deleted:
        continue

      # Omit other OS's files since we can't know if they were deleted (__os__ files
      # and a few miscellaneous ones that are OS-specific but not in __os__ dirs)
      if '__os__' in ditem and '__os__/' + osname not in ditem:
        deleted.remove(ditem)
      # XXX Would be better to pass the below in; see also GenerateUpdate which has
      # XXX related model for what's OS-speciifc
      elif ditem in ('src/wingapi.pyc', 'build-files/wing-install.py', 'wingdb.exe'):
        deleted.remove(ditem)
      elif ditem.startswith('bin/') and ditem[len('bin/'):].find('/') == -1 and \
           ditem.endswith(('.exe', '.dll')):
        deleted.remove(ditem)
        
      # Omit un-updateable files which should never be deleted
      elif not ok_to_include(ditem):
        deleted.remove(ditem)
  else:
    deleted = set()
    
  # Compute new catalog, which should change only entries which
  # we determined to be different.  This is needed to prevent changing
  # the digest of items that did not change according to source digest
  # (the file itself always changes, but the source has not so we are
  # keeping the old build target).  Also remove any deleted items.
  new_catalog = old.copy()
  for fp in changed_files_in_catalog:
    new_catalog[fp] = new[fp]
  for fp in deleted:
    del new_catalog[fp]
  
  return changed_files, delete_first, deleted, new_catalog, omitted_but_changed

#----------------------------------------------------------------------
def ParseVersionString(ver):
  """Convert #.#.#.# string into (#, #, #, #) tuple of ints"""
  
  ver = ver.split('.')
  if len(ver) != 4:
    return None
  try:
    return tuple([int(v) for v in ver])
  except:
    return None
    
#----------------------------------------------------------------------
def ParseReleases(txt, from_version, to_version, include_from=False):
  """Parse RELEASES.txt formatted release list for all releases between from_version
  and to_version, not including from_version."""

  summaries = collections.defaultdict(list)
  version_info = {}
  version = None
  skip_version = False
  for i, line in enumerate(txt.splitlines()):
    
    if line.startswith('#') or not line.strip():
      continue
    
    elif line.startswith('==='):
      parts = line.split()
      if len(parts) < 3:
        return _("Version spec on line %i is too short") % i, None, None
      version = parts[1]
      version_tuple = ParseVersionString(version)
      if version_tuple is None:
        return _("Bad version %s on line %i") % (version, i), None, None
      if not include_from and version_tuple <= from_version:
        return None, summaries, version_info
      if include_from and version_tuple < from_version:
        return None, summaries, version_info
      if version_tuple > to_version:
        skip_version = True
        continue
      else:
        skip_version = False
      rtype = parts[2].lower()
      if rtype not in ['alpha', 'beta', 'rc', 'final']:
        return _("Bad release type %s on line %i") % (rtype, i), None, None
      flags = parts[3:]
      for flag in flags:
        if flag not in ['force-download', 'patch']:
          return _("Bad release flag %s on line %i") % (flag, i), None, None
      version_info[version_tuple] = (rtype, flags)
        
    elif not skip_version:
      if version is None:
        return _("Line %i lists summary before version") % i, None, None
      parts = line.split()
      if len(parts) < 4:
        return _("Bad change summary on line %i") % i, None, None
      prod = parts[0].lower()
      if prod not in ['all', 'per', 'pro']:
        return _("Bad product type on line %i") % i, None, None
      ctype = parts[1].lower()
      if ctype not in ('f', 'c', 'b'):
        return _("Bad change type on line %i") % i, None, None
      change = ' '.join(parts[2:])
      summaries[version_tuple].append((parts[0], parts[1], change))
      
  return None, summaries, version_info

#----------------------------------------------------------------------
def FindAvailableReleases(prod, from_version, releases_fn, max_version=None):
  """Find all available releases for the given version by reading the given RELEASES.txt
  file. This takes into account whether or not a release can be applied with the update
  manager, and whether a release has installers. Returns latest_update, newer_release."""
  
  # Note: Only used on the website side but kept here so we have identical
  # code on both ends (could be reorganized but...)
  
  def relevant_changes(prod, changes):
    for cprod, ctype, change in changes:
      cprod = cprod.lower()
      if cprod == 'all' or prod == 'wingpro' or ((prod == 'wing-personal' and cprod in ('per', '101')) or
                                                 (prod == 'wing-101' and cprod == '101')):
        return True
      
    return False
      
  f = open(releases_fn, 'rb')
  content_bytes = f.read()
  f.close()
  
  if sys.version_info >= (3, 0):
    txt = content_bytes.decode('utf-8')
  else:
    txt = content_bytes

  if max_version is None:
    max_version = (1000, 1000, 1000, 1000)
    
  err, releases, version_info = ParseReleases(txt, from_version, max_version,
                                              include_from=True)
  if not releases:
    return None, None
  
  try:
    from_rtype, from_flags = version_info[from_version]
  except KeyError:
    from_rtype = 'alpha'
    from_flags = []
  
  update_candidates = []
  newer_release = None
  for ver in reversed(sorted(releases)):
    if ver == from_version:
      break
    rtype, flags = version_info[ver]
    if 'force-download' in flags:
      update_candidates = []
    # Don't consider prereleases if not running one
    if rtype in ('alpha', 'beta', 'rc') and from_rtype not in ('alpha', 'beta', 'rc'):
      continue
    # Ignore releases with no changes for selected product
    if not relevant_changes(prod, releases[ver]):
      continue
    # Save latest release with installers
    if newer_release is None and 'patch' not in flags:
      newer_release = ver
    # Find latest release to which we can update without an installer
    if 'force-download' not in flags:
      update_candidates.append(ver)

  update_candidates.sort()
  if update_candidates:
    update_version = update_candidates[-1]
  else:
    update_version = None

  if newer_release == update_version:
    newer_release = None
  
  return update_version, newer_release
  
#----------------------------------------------------------------------
def GenerateUpdate(from_version, to_version, install_root, osname, from_catalog,
                   to_catalog, target_file, build_utils, omitted=None):
  """Generate an update file from the given install root"""

  # Caller may send in some already-omitted items; this is used so we can omit all binaries
  # in an update, and thus build it on any machine and not necessarily a build VM
  if omitted is None:
    omitted = {}

  # Never update anything in bootstrap or selected files at top level that are
  # actually OS-specific but outside of __os__ dirs
  # Updating these requires making a new release with force-download set in RELEASES.txt
  omitted.update({
    'bootstrap/': ('*',),
    '/': ('wing', 'wing-personal', 'wing-101', '*.exe'),
    'bin': ('*.exe', '*.dll'),
  })
  
  # Omit *.pyo files for a handful of modules we distribute as *.py
  omitted['bin/'] = ('*/execonearg.pyo', '*/updateutils.pyo', '*/mergeimporter.pyo', '*/vinfo.pyo')
  omitted['src/'] = ('*/wingapi.pyo', )
  
  # Before 7.1.2.4 there were no source hashes for these so had this hack to avoids
  # always updating them; in 7.1.2.4 we let them all update
  to_version_tuple = tuple(int(i) for i in to_version.split('.'))
  if to_version_tuple < (7, 1, 2, 4):
    omitted['resources/builtin-pi-files'] = ('*', )
  
  # XXX For now, omit runtimes; later need to include whole dir if the
  # XXX version number changes
  runtimes_dir = install_root+'/bin/__os__/%s/' % osname
  if os.path.exists(runtimes_dir):
    for fn in os.listdir(runtimes_dir):
      if fn.startswith('runtime-'):
        omitted['bin/__os__/%s/%s/' % (osname, fn)] = ('*',)
  
  # Include whole directories for some things if anything changes in them
  whole_dirs = [
    'src/testing/runners/',
    'src/external/pygments/',
    'plugins/ai/',
    'resources/icons', 
    'resources/typeshed/', 
    'resources/typing/',
    'resources/syntax/', 
    'resources/builtin-pi-files/',
    'resources/package-pi-files/',
    'resources/locale/', 
    'tutorial/',
  ]
  external_packages_dir = (
    install_root + '/bin/__os__/%s/external-packages' % (osname,)
  )
  if os.path.isdir(external_packages_dir):
    for fn in os.listdir(external_packages_dir):
      fullname = external_packages_dir + '/' + fn
      if os.path.isdir(fullname):
        whole_dirs.append(fullname[len(install_root+'/'):])
  
  # Find changed or new files and deleted files
  changed_or_new, delete_first, deleted, new_catalog, omitted_but_changed = \
    CompareDirectoryCatalogs(install_root, from_catalog, to_catalog, osname, whole_dirs, omitted)
  if omitted_but_changed:
    warning = _("Some unpatchable files changed:"), ', '.join(omitted_but_changed)
  else:
    warning = None
  if not changed_or_new and not deleted:
    return _("No changes for the update"), warning
  
  # Make a temporary directory with the changed files.  We skip over files for a
  # different OS, except if they are for the debugger / remote agent since those
  # are used by the IDE to install on a remote host.
  upload_tmp = tempfile.mkdtemp() + '/' + to_version
  build_utils.EnsureDirExists(upload_tmp+'/xxx')
  for fp in changed_or_new:
    if '__os__' in fp and not '__os__/' + osname in fp and 'bin/dbg/src' not in fp:
      continue
    ifp = install_root + '/' + fp
    tfp = upload_tmp + '/' + fp
    build_utils.EnsureDirExists(tfp)
    build_utils.CopyFileIfNewer(ifp, tfp, force=1, recursive=1)
    
  # Also write the generated catalog from CompareDirectoryCatalogs, which changes
  # entries only for files that actually changed (as opposed to just being
  # different; for many files this is determined by looking at source hashes)
  WriteDirectoryCatalog(new_catalog, upload_tmp+'/CATALOG.txt')

  # Add a registry of deleted files
  f = open(upload_tmp+'/DELETED.txt', 'wb')
  contents = '\n'.join(sorted(deleted))
  f.write(contents.encode('utf-8'))
  f.close()

  # Create update tarfile
  temp_tar = tempfile.mktemp(suffix='.tar.bz2')
  f = tarfile.TarFile.open(temp_tar, 'w:bz2')
  f.add(upload_tmp, to_version)
  f.close()
  build_utils.Delete(upload_tmp, force=1)
  if not os.path.exists(temp_tar):
    return _("Could not create update tarfile:"), warning

  # Copy it into place
  build_utils.EnsureDirExists(target_file)
  build_utils.CopyFileIfNewer(temp_tar, target_file)
  os.unlink(temp_tar)
  
  return None, warning

#-----------------------------------------------------------------------
def Delete(name, recursive=True, force=True):
  """ Delete files or directories (if recursive is true).  Returns whether it
  was able to delete everything; it tries to delete as much as it can even
  if it encounters errors.  This avoids bugs in shutil.rmtree that prevent
  removing read-only files."""
  
  deleted_all = True
  
  if os.path.isfile(name) or os.path.islink(name):
    if force and not os.path.islink(name):
      try:
        os.chmod(name, 438)  # octal 0666
      except OSError:
        pass
    try:  
      os.remove(name)
    except OSError:
      deleted_all = False
      
  elif os.path.isdir(name) and recursive:
    if force:
      try:
        os.chmod(name, 511)  # octal 0777
      except OSError:
        pass

    try:
      children = os.listdir(name)
    except OSError:
      children = ()
      deleted_all = False
    for child in children:
      sub_all = Delete(os.path.join(name, child), recursive, force)
      if not sub_all:
        deleted_all = False
        
    try:
      os.rmdir(name)
    except OSError:
      deleted_all = False
      
  return deleted_all

#----------------------------------------------------------------------
def RemoveOldUpdates(update_roots, install_version):
  """Try to remove all old update directories"""
  
  failed = []
  for update_root in update_roots:
    
    try:
      name_list = os.listdir(update_root)
    except:
      continue
    
    for name in name_list:
      remove = False
      fp = update_root + '/' + name

      # Update parent directory written by Wing 8.3.3+; the name indicates the
      # target install version and we remove it if it does not match the
      # running version of Wing
      if name.startswith('from_'):
        ver = NameToVersion(name[len('from_'):])
        remove = ver is not None and ver < install_version

      # Update written by Wing <8.3.3; the name is the version being updated to
      # This had the problem that downgrading install version broke things and
      # we remove anything named #.#.#.# since they are no longer needed in 8.3.3+
      else:
        ver = NameToVersion(name)
        remove = ver is not None
        
      if remove:
        
        try:
          # Try rename first so it fails on win32 if any file is in use by another instance;
          # this also prevents leaving a partially deleted update that Wing may try to load
          os.rename(fp, fp+'-old')
          fp += '-old'
          Delete(fp)
        except:
          failed.append(fp)
    
  return failed
  
#-----------------------------------------------------------------------
def MarkUpdatesCorrupt(update_dirs):

  all_marked = True
  
  for update_dir in update_dirs:
    if not os.path.exists(update_dir):
      continue
    try:
      f = open(os.path.join(update_dir, 'CORRUPT.txt'), 'wb')
      contents = _("Updates are corrupt and need to be removed and reinstalled") + '\n'
      f.write(contents.encode('utf-8'))
      f.close()
    except:
      all_marked = False
      
  return all_marked
        
#----------------------------------------------------------------------
kMaxUpdateDirs = 3
def MergeUpdateDirs(code_dirs):
  """Try to merge older update directories.  Returns (fatal, error_list).
  When fatal is true, the updates are corrupted and should not be loaded."""
  
  errors = []
  
  # Omit WINGHOME and change to oldest first sorting
  code_dirs = code_dirs[:-1]
  code_dirs.reverse()

  # Assumption made by the implementation
  assert kMaxUpdateDirs >= 2
  
  if len(code_dirs) <= kMaxUpdateDirs:
    return False, []

  fatal_errors = []
  def try_fatal_rename(old, new):
    """Try a rename that on failure will corrupt the update area.  If it fails,
    we start without updates, although subsequent starts will likely fail entirely."""
    try:
      os.rename(old, new)
      return True
    except:
      fatal_errors.append((old, new))
      return False
    
  # Rename all the directories we're going to merge from/to all at once to prevent
  # concurrent update/merge by multiple instances of Wing.
  renamed = set()
  for from_ver, from_dir in code_dirs[:-kMaxUpdateDirs+1]:
    from_dir_busy = from_dir+'-busy'
    try:
      
      # Wait up to 2 hours before removing an existing -busy directory in case
      # it's due to a concurrent update/merge process
      if os.path.exists(from_dir_busy):
        mtime = os.stat(from_dir_busy)[stat.ST_MTIME]
        if mtime < time.time() - 2 * 60 * 60:
          if not Delete(from_dir_busy):
            errors.append(_("Failed to delete old busy update: %s") % from_dir_busy)
            raise RuntimeError("failed-delete")
        else:
          errors.append(_("Possibly concurrent update/merge on %s") % from_dir_busy)
          raise RuntimeError('concurrent-update-merge')
        
      # Try to rename it; fails on Windows if in use by another process
      os.rename(from_dir, from_dir_busy)
      renamed.add(from_dir)
      
    except:
      errors.append(_("Renaming %s failed; skipping update merge"))
        
      # Try to put back the dirs we already renamed; fatal if it fails
      for dirname in renamed:
        try_fatal_rename(dirname+'-busy', dirname)
      
      return bool(fatal_errors), errors
  
  # Merge oldest update directories one at a time; any that fail are left alone
  # if possible under their original name (w/o -busy)
  while len(code_dirs) > kMaxUpdateDirs:
    
    # These were all renamed above
    from_dir = code_dirs[0][1] + '-busy'
    to_dir = code_dirs[1][1] + '-busy'
    tmp_to_dir = to_dir + '-merge'
    
    try:
        
      # Copy the target directory to a temp directory so we can replace it atomically
      shutil.copytree(to_dir, tmp_to_dir)
      
      # Merge into temp dir and remove from_dir if is succeeds
      MergeUpdateDir(from_dir, tmp_to_dir)
      
      # Move temp merge dir into place and remove old dir
      if try_fatal_rename(to_dir, to_dir+'-old'):
        if try_fatal_rename(tmp_to_dir, to_dir):
          Delete(to_dir+'-old')
        else:
          try_fatal_rename(to_dir+'-old', to_dir)
      
    except:
      errors.append(_("Failed to merge update directories: %s -> %s; aborting merge") % (from_dir, to_dir))
      errors.append(traceback.format_exc())
      Delete(tmp_to_dir)
      if os.path.exists(from_dir):
        try_fatal_rename(from_dir, from_dir[:-len('-busy')])
    
    if fatal_errors:
      return True, errors
    
    code_dirs = code_dirs[1:]
    
  # Rename the final target directory; this is fatal if it fails
  if try_fatal_rename(to_dir, to_dir[:-len('-busy')]):
    return False, errors
  else:
    return True, errors

#----------------------------------------------------------------------
def MergeUpdateDir(from_dir, to_dir):
  """Merge one update directory into another, updating files in the to_dir as
  necessary so that to_dir may be used without from_dir. Note that the
  CATALOG.txt in to_dir is not updated """
  
  def read_deleted_set(deleted_fn):
    # Read deleted files for newer version
    try:
      f = open(deleted_fn, 'rb')
      contents = f.read()
      f.close()
    except:
      return set()
    
    if sys.version_info < (3, 0):
      txt = contents
    else:
      txt = contents.decode('utf-8')
    line_list = [l.strip() for l in txt.splitlines()]
    to_del = [l for l in line_list if l != '']
      
    return set(to_del)
    
    

  if from_dir.endswith(os.sep):
    from_dir = from_dir[:-1]
  if to_dir.endswith(os.sep):
    to_dir = to_dir[:-1]
    
  # Read deleted files for newer version
  to_del_set = read_deleted_set(os.path.join(to_dir, 'DELETED.txt'))
  
  # Get list of files in newer version
  newer_files = set()
  for dirname, dirs, files in os.walk(to_dir):
    for fn in files:
      fullpath = os.path.join(dirname, fn)
      relpath = fullpath[len(to_dir)+1:].replace(os.sep, '/')
      newer_files.add(relpath)
    
  # Copy over all files except: DELETED.txt, any since-deleted files,
  # and any files already present in newer update
  for dirname, dirs, files in os.walk(from_dir):
    for fn in files:
      if fn == 'DELETED.txt' and dirname == from_dir:
        continue
      fullpath = os.path.join(dirname, fn)
      relpath = fullpath[len(from_dir)+1:]
      if relpath in to_del_set:
        continue
      target = os.path.join(to_dir, relpath)
      if os.path.exists(target):
        continue
      if not os.path.exists(os.path.dirname(target)):
        # mode=0755 octal; plain int form works in all Python versions:
        os.makedirs(os.path.dirname(target), mode=493)
      shutil.copy2(fullpath, os.path.join(to_dir, relpath))
    
  # Read old DELETED.txt
  old_deleted_set = read_deleted_set(os.path.join(from_dir, 'DELETED.txt'))
  
  merged_deleted_set = to_del_set.union(old_deleted_set)

  # Remove any items that have been added again
  merged_deleted_set.difference_update(newer_files)

  # Write merged DELETED.txt into newer dir
  f = open(os.path.join(to_dir, 'DELETED.txt'), 'wb')
  contents = '\n'.join(sorted(merged_deleted_set))
  f.write(contents.encode('utf-8'))
  f.close()
  
  # Remove old update dir
  Delete(from_dir)
  
#----------------------------------------------------------------------
def ValidateInstall(code_dirs, ignores, progress_cb=None):
  """Validate that all the files in updates match the checksums in the latest catalog
  for the given code directories."""

  # This is currently used only in the IDE
  import bootstrap_utils
  osname = bootstrap_utils.NP_GetPlatform()
  
  # Some files to ignore because the installer changes them or they get rewritten
  # at runtime
  ignores.extend([
    'wingdbstub.py',
    'src/wingapi.pyo'
  ])
  
  def get_file_os(fn):
    
    # Determine OS for the file based on standard layout
    if '__os__' in fn:
      fn = fn[fn.find('__os__'):][len('__os__') + 1:]
      if not fn:
        return None
      fos = fn[fn.find('/')]
      return fos

    # A few files are placed at top level or in bin or bootstrap but are OS-specific
    # so need to be ignored on other OSes
    osfiles = [
      ('', ('wing', 'wing-personal', 'wing-101', 'wingdb-suid'), ('linux', 'macos')), 
      ('', ('wingdb.exe', ), ('win32',)), 
      ('', ('wing.icns',), ('macos',)), 
      ('bin', ('wing*.exe', 'console*.exe', '*.dll'), ('win32',)),
      ('bootstrap', ('run-wing.sh', ), ('linux', 'macos')), 
    ]
    for prefix, patterns, oses in osfiles:
      if fn.startswith(prefix):
        nfn = fn[len(prefix):]
        if nfn.startswith('/'):
          nfn = nfn[1:]
        parts = nfn.split('/')
        for pat in patterns:
          if fnmatch.fnmatch(parts[0], pat):
            return oses
          
    # File is for all OSes
    return None
            
  latest_ver, latest_dir = code_dirs[0]
  fn = os.path.join(latest_dir, 'CATALOG.txt')
  error, catalog = ReadDirectoryCatalog(fn)

  if callable(progress_cb):
    progress_cb(0, len(catalog))
    
  if error:
    return [error]

  def ignore(pp):
    for pat in ignores:
      if fnmatch.fnmatch(pp, pat):
        return True
    return False
  
  def find_file(pp):
    assert not os.path.isabs(pp)
    for cv, cdir in code_dirs:
      fn = os.path.join(cdir, pp)
      if os.path.exists(fn):
        return fn
    return None

  errors = []
  start_time = time.time()
  for i, partial_path in enumerate(catalog):
    if callable(progress_cb) and time.time() > start_time + 1.0:
      start_time = time.time()
      progress_cb(i, len(catalog))
    file_type, digest, source_digest = catalog[partial_path]
    fn = find_file(partial_path)
    if fn is None:
      # Ignore missing files for other OSes
      foses = get_file_os(partial_path)
      if foses is None or osname in foses:
        errors.append((partial_path, _("Missing file: %s") % partial_path))
    elif ignore(partial_path):
      continue
    elif file_type == 'F' and not os.path.isfile(fn):
      errors.append((partial_path, _("Invalid Type: %s (expected file but found non-file)") % partial_path))
    elif file_type == 'D' and not os.path.isdir(fn):
      errors.append((partial_path, _("Invalid Type: %s (expected directory but found non-directory)") % partial_path))
    elif file_type == 'F':
      try:
        file_digest = GetSHA1Digest([fn], normalize_eol=True)
      except:
        errors.append((partial_path, _("Read failed:  %s") % partial_path))
      else:
        if file_digest != digest:
          errors.append((partial_path, _("Changed file: %s") % partial_path))
        
  errors.sort()
  errors = [e[1] for e in errors]
  
  return errors
    
#-----------------------------------------------------------------------
def NameToVersion(name):
  """Try to convert given directory name to a 4-int version tuple.  Returns
  None if it's not a version directory."""

  parts = name.split('.')
  if len(parts) not in (3, 4):
    return None
  try:
    retval = tuple([int(p) for p in parts])
    if len(retval) == 3:
      return retval + (0, )
    else:
      return retval
  except:
    return None

