Python log incremental capture implementation method

Below I will share with you an implementation method of incremental log capture in Python, which has good reference value and I hope it will be helpful to you. Let’s take a look together

The example is as follows:

import time
import pickle
import os
import re
class LogIncScaner(object):
  def __init__(self,log_file, reg_ex,seek_file='/tmp/log-inc-scan.seek.temp'):
    self.log_file = log_file
    self.reg_ex = reg_ex
    self.seek_file = seek_file
  def scan(self):
    seek = self._get_seek()
    file_mtime = os.path.getmtime(self.log_file)
    if file_mtime <= seek[&#39;time&#39;]:
      print &#39;file mtime not change since last scan&#39;
      seek[&#39;time&#39;] = file_mtime
      self._dump_seek(seek)
      return []
    file_size = os.path.getsize(self.log_file)
    if file_size <= seek[&#39;position&#39;]:
      print &#39;file size not change since last scan&#39;
      seek[&#39;position&#39;] = file_size
      self._dump_seek(seek)
      return []
    print &#39;file changed,start to scan&#39;
    matchs = []
    with open(self.log_file, &#39;rb&#39;) as logfd:
      logfd.seek(seek[&#39;position&#39;],os.SEEK_SET)
      for match in re.finditer(self.reg_ex, logfd.read()):
        matchs.append(match)
      seek = {&#39;time&#39;:time.time(),&#39;position&#39;: logfd.tell()}
      print seek
      self._dump_seek(seek)
    return matchs
  def _get_seek(self):
    seek = {&#39;time&#39;:time.time(),&#39;position&#39;:0}
    if os.path.exists(self.seek_file):
      with open(self.seek_file,&#39;rb&#39;) as seekfd:
          try:
            seek = pickle.load(seekfd)
          except:
            pass
    print seek
    return seek
  def _dump_seek(self, seek):
    with open(self.seek_file,&#39;wb&#39;) as seekfd:
      pickle.dump(seek,seekfd)
  def reset_seek(self):
    self._dump_seek({&#39;time&#39;:time.time(),&#39;position&#39;:0})
if __name__ == "__main__":
  scaner = LogIncScaner(&#39;/var/log/messages&#39;,r&#39;(\w+ \d+ \d+:\d+:\d+) .+?exception&#39;)
  scaner.reset_seek()
  while True:
    matchs = scaner.scan()
    for match in matchs:
      print &#39;fond at:&#39; + match.group(1) + &#39; content:&#39; + match.group(0)
    time.sleep(5)

Related recommendations:

A brief discussion on the configuration file path problem of python logs

Detailed explanation of python log printing and writing concurrency implementation code

The above is the detailed content of Python log incremental capture implementation method. For more information, please follow other related articles on the PHP Chinese website!