[chirp_devel] [PATCH] RFC [csv] Robust CSV support, fixes regression from #477

[Tom Hayward]

# HG changeset patch
# User Tom Hayward <tom at tomh.us>
# Date 1411589476 25200
#      Wed Sep 24 13:11:16 2014 -0700
# Node ID df29a2e54b5df6831c4b02df326f00fd77054a96
# Parent  79d6fef85d326995fb7d44bd0179332c63068c46
RFC [csv] Robust CSV support, fixes regression from #477

#477 changed CSV detection so that Chirp CSVs must start with "Location,".
Python's CSV reader is capable of parsing many other variations, like quoted
values and semicolon separators. Should we support these?

Also, I could not find any reason to return an error for a column number
mismatch (no commas at end of line). Our CSV algorithm doesn't care.

I'll break this into multiple patches after comments.

Tom KD7LXL

diff -r 79d6fef85d32 -r df29a2e54b5d chirp/generic_csv.py
--- a/chirp/generic_csv.py	Sun Sep 14 12:35:46 2014 +0200
+++ b/chirp/generic_csv.py	Wed Sep 24 13:11:16 2014 -0700
@@ -170,10 +170,10 @@
         self._blank()
 
         f = file(self._filename, "rU")
-        header = f.readline().strip()
 
+        dialect = csv.Sniffer().sniff(f.read(1024))
         f.seek(0, 0)
-        reader = csv.reader(f, delimiter=chirp_common.SEPCHAR, quotechar='"')
+        reader = csv.reader(f, dialect=dialect)
 
         good = 0
         lineno = 0
@@ -185,13 +185,6 @@
                 self.file_has_cTone = "cToneFreq" in header
                 continue
 
-            if len(header) > len(line):
-                print "Line %i has %i columns, expected %i" % (lineno,
-                                                               len(line),
-                                                               len(header))
-                self.errors.append("Column number mismatch on line %i" % lineno)
-                continue
-
             try:
                 mem = self._parse_csv_data_line(header, line)
                 if mem.number is None:
@@ -272,8 +265,21 @@
     @classmethod
     def match_model(cls, filedata, filename):
         """Match files ending in .CSV"""
-        return filename.lower().endswith("." + cls.FILE_EXTENSION) and \
-            (filedata.startswith("Location,") or filedata == "")
+        if filename.lower().endswith("." + cls.FILE_EXTENSION):
+            if filedata == "":
+                # File > New
+                return True
+            try:
+                from StringIO import StringIO
+                dialect = csv.Sniffer().sniff(filedata)
+                reader = csv.reader(StringIO(filedata), dialect=dialect)
+                for row in reader:
+                    if all(["Location" in row, "Frequency" in row]):
+                        return True
+                    return False
+            except csv.Error:
+                return False
+        return False
 
 
 @directory.register