From 4ee7ec2127a240692f6566f8840868d5405bb774 Mon Sep 17 00:00:00 2001 From: "Erik C. Thauvin" Date: Sat, 16 Oct 2021 00:54:06 -0700 Subject: [PATCH] Rewrote data parsing using streams. --- .gitignore | 2 +- bin/dcat.dart | 114 +++++++++++++----------- lib/dcat.dart | 211 +++++++++++++++++++++----------------------- pubspec.lock | 7 ++ pubspec.yaml | 1 + test/dcat_test.dart | 161 ++++++++++++++++++++------------- 6 files changed, 273 insertions(+), 223 deletions(-) diff --git a/.gitignore b/.gitignore index 741e785..9ef9eee 100644 --- a/.gitignore +++ b/.gitignore @@ -51,7 +51,6 @@ /**/.idea/**/workspace.xml /**/.idea/sonarlint* /**/.idea_modules/ -/coverage/ Thumbs.db __pycache__ atlassian-ide-plugin.xml @@ -60,6 +59,7 @@ bin/dcat.exe build/ cmake-build-*/ com_crashlytics_export_strings.xml +coverage/ crashlytics-build.properties crashlytics.properties dependency-reduced-pom.xml diff --git a/bin/dcat.dart b/bin/dcat.dart index 4626893..7820519 100644 --- a/bin/dcat.dart +++ b/bin/dcat.dart @@ -11,8 +11,8 @@ import 'package:indent/indent.dart'; const appName = 'dcat'; const appVersion = '1.0.0'; const helpFlag = 'help'; -const nonBlankFlag = 'number-nonblank'; const numberFlag = 'number'; +const numberNonBlank = 'number-nonblank'; const showAllFlag = 'show-all'; const showEndsFlag = 'show-ends'; const showNonPrintingEndsFlag = 'show-nonprinting-ends'; @@ -26,13 +26,71 @@ const versionFlag = 'version'; /// /// Usage: `dcat [option] [file]…` Future main(List arguments) async { - final parser = ArgParser(); - exitCode = exitSuccess; + final parser = await setupArgsParser(); + final ArgResults argResults; + try { + argResults = parser.parse(arguments); + } on FormatException catch (e) { + return printError( + "${e.message}\nTry '$appName --$helpFlag' for more information."); + } + + if (argResults[helpFlag]) { + exitCode = await usage(parser.usage); + } else if (argResults[versionFlag]) { + exitCode = await printVersion(); + } else { + final paths = argResults.rest; + var showEnds = argResults[showEndsFlag]; + var showTabs = argResults[showTabsFlag]; + var showLineNumbers = argResults[numberFlag]; + var showNonBlank = argResults[numberNonBlank]; + var showNonPrinting = argResults[showNonPrintingFlag]; + + if (argResults[showNonPrintingEndsFlag]) { + showNonPrinting = showEnds = true; + } + + if (argResults[showNonPrintingTabsFlag]) { + showNonPrinting = showTabs = true; + } + + if (argResults[showAllFlag]) { + showNonPrinting = showEnds = showTabs = true; + } + + if (showNonBlank) { + showLineNumbers = true; + } + + final result = await cat(paths, stdout, + input: stdin, + showEnds: showEnds, + showLineNumbers: showLineNumbers, + numberNonBlank: showNonBlank, + showTabs: showTabs, + squeezeBlank: argResults[squeezeBlank], + showNonPrinting: showNonPrinting); + + for (final message in result.messages) { + await printError(message); + } + + exitCode = result.exitCode; + } + + return exitCode; +} + +/// Setup the command-line arguments parser. +Future setupArgsParser() async { + final parser = ArgParser(); + parser.addFlag(showAllFlag, negatable: false, abbr: 'A', help: 'equivalent to -vET'); - parser.addFlag(nonBlankFlag, + parser.addFlag(numberNonBlank, negatable: false, abbr: 'b', help: 'number nonempty output lines, overrides -n'); @@ -60,53 +118,7 @@ Future main(List arguments) async { abbr: 'v', help: 'use ^ and U+ notation, except for LFD and TAB'); - final ArgResults argResults; - try { - argResults = parser.parse(arguments); - } on FormatException catch (e) { - return printError( - "${e.message}\nTry '$appName --$helpFlag' for more information."); - } - - if (argResults[helpFlag]) { - exitCode = await usage(parser.usage); - } else if (argResults[versionFlag]) { - exitCode = await printVersion(); - } else { - final paths = argResults.rest; - var showEnds = argResults[showEndsFlag]; - var showTabs = argResults[showTabsFlag]; - var showNonPrinting = argResults[showNonPrintingFlag]; - - if (argResults[showNonPrintingEndsFlag]) { - showNonPrinting = showEnds = true; - } - - if (argResults[showNonPrintingTabsFlag]) { - showNonPrinting = showTabs = true; - } - - if (argResults[showAllFlag]) { - showNonPrinting = showEnds = showTabs = true; - } - - final result = await cat(paths, stdout, - input: stdin, - showEnds: showEnds, - showLineNumbers: argResults[numberFlag], - numberNonBlank: argResults[nonBlankFlag], - showTabs: showTabs, - squeezeBlank: argResults[squeezeBlank], - showNonPrinting: showNonPrinting); - - for (final message in result.messages) { - await printError(message); - } - - exitCode = result.exitCode; - } - - return exitCode; + return parser; } /// Prints the error [message] to [stderr]. diff --git a/lib/dcat.dart b/lib/dcat.dart index 6ab16ce..8ce9daa 100644 --- a/lib/dcat.dart +++ b/lib/dcat.dart @@ -8,19 +8,31 @@ library dcat; import 'dart:convert'; import 'dart:io'; +/// Failure exit code. const exitFailure = 1; + +/// Success exit code. const exitSuccess = 0; +const _lineFeed = 10; + /// Holds the [cat] result [exitCode] and error [messages]. class CatResult { /// The exit code. int exitCode = exitSuccess; + /// The error messages. final List messages = []; CatResult(); - /// Add a message. + /// Returns `true` if the [exitCode] is [exitFailure]. + bool get isFailure => exitCode == exitFailure; + + /// Returns `true` if the [exitCode] is [exitSuccess]. + bool get isSuccess => exitCode == exitSuccess; + + /// Add a message with an optional path. void addMessage(int exitCode, String message, {String? path}) { this.exitCode = exitCode; if (path != null && path.isNotEmpty) { @@ -31,40 +43,35 @@ class CatResult { } } -/// Concatenates files in [paths] to [stdout] or [File]. +// Holds the current line number and last character. +class _LastLine { + int lineNumber; + int lastChar; + + _LastLine(this.lineNumber, this.lastChar); +} + +/// Concatenates files in [paths] to the standard output or a file. /// -/// * [output] should be an [IOSink] like [stdout] or a [File]. +/// * [output] should be an [IOSink] such as [stdout] or [File.openWrite]. /// * [input] can be [stdin]. -/// * [log] is used for debugging/testing purposes. /// /// The remaining optional parameters are similar to the [GNU cat utility](https://www.gnu.org/software/coreutils/manual/html_node/cat-invocation.html#cat-invocation). -Future cat(List paths, Object output, +Future cat(List paths, IOSink output, {Stream>? input, - List? log, bool showEnds = false, bool numberNonBlank = false, bool showLineNumbers = false, bool showTabs = false, bool squeezeBlank = false, bool showNonPrinting = false}) async { - var result = CatResult(); - var lineNumber = 1; - log?.clear(); + final result = CatResult(); + final lastLine = _LastLine(0, _lineFeed); if (paths.isEmpty) { if (input != null) { - final lines = await _readStream(input); try { - await _writeLines( - lines, - lineNumber, - output, - log, - showEnds, - showLineNumbers, - numberNonBlank, - showTabs, - squeezeBlank, - showNonPrinting); + await _writeStream(input, lastLine, output, showEnds, showLineNumbers, + numberNonBlank, showTabs, squeezeBlank, showNonPrinting); } catch (e) { result.addMessage(exitFailure, '$e'); } @@ -72,25 +79,14 @@ Future cat(List paths, Object output, } else { for (final path in paths) { try { - final Stream lines; + final Stream> stream; if (path == '-' && input != null) { - lines = await _readStream(input); + stream = input; } else { - lines = utf8.decoder - .bind(File(path).openRead()) - .transform(const LineSplitter()); + stream = File(path).openRead(); } - lineNumber = await _writeLines( - lines, - lineNumber, - output, - log, - showEnds, - showLineNumbers, - numberNonBlank, - showTabs, - squeezeBlank, - showNonPrinting); + await _writeStream(stream, lastLine, output, showEnds, showLineNumbers, + numberNonBlank, showTabs, squeezeBlank, showNonPrinting); } on FileSystemException catch (e) { final String? osMessage = e.osError?.message; final String message; @@ -111,80 +107,77 @@ Future cat(List paths, Object output, return result; } -/// Parses line with non-printing characters. -Future _parseNonPrinting(String line, bool showTabs) async { +// Writes parsed data from a stream +Future _writeStream( + Stream stream, + _LastLine lastLine, + IOSink out, + bool showEnds, + bool showLineNumbers, + bool numberNonBlank, + bool showTabs, + bool squeezeBlank, + bool showNonPrinting) async { + const tab = 9; + int squeeze = 0; final sb = StringBuffer(); - for (var ch in line.runes) { - if (ch >= 32) { - if (ch < 127) { - sb.writeCharCode(ch); - } else if (ch == 127) { - sb.write('^?'); - } else { - sb.write('U+' + ch.toRadixString(16).padLeft(4, '0').toUpperCase()); - } - } else if (ch == 9 && !showTabs) { - sb.write('\t'); - } else { - sb - ..write('^') - ..writeCharCode(ch + 64); - } - } - return sb.toString(); -} - -/// Reads from stream (stdin, etc.) -Future> _readStream(Stream> input) async => - input.transform(utf8.decoder).transform(const LineSplitter()); - -/// Writes lines to stdout. -Future _writeLines(Stream lines, int lineNumber, Object out, - [List? log, - bool showEnds = false, - bool showLineNumbers = false, - bool showNonBlank = false, - bool showTabs = false, - bool squeezeBlank = false, - bool showNonPrinting = false]) async { - var emptyLine = 0; - final sb = StringBuffer(); - await for (final line in lines) { + await stream.forEach((data) { sb.clear(); - if (squeezeBlank && line.isEmpty) { - if (++emptyLine >= 2) { - continue; + for (final ch in utf8.decode(data).runes) { + if (lastLine.lastChar == _lineFeed) { + if (squeezeBlank) { + if (ch == _lineFeed) { + if (squeeze >= 1) { + lastLine.lastChar = ch; + continue; + } + squeeze++; + } else { + squeeze = 0; + } + } + if (showLineNumbers || numberNonBlank) { + if (!numberNonBlank || ch != _lineFeed) { + sb.write('${++lastLine.lineNumber}'.padLeft(6) + '\t'); + } + } } - } else { - emptyLine = 0; - } - if (showLineNumbers || (showNonBlank && line.isNotEmpty)) { - sb.write('${lineNumber++} '.padLeft(8)); - } - - if (showNonPrinting) { - sb.write(await _parseNonPrinting(line, showTabs)); - } else if (showTabs) { - sb.write(line.replaceAll('\t', '^I')); - } else { - sb.write(line); - } - - if (showEnds) { - sb.write('\$'); - } - - log?.add(sb.toString()); - - try { - if (out is IOSink) { - out.writeln(sb); - } else if (out is File) { - await out.writeAsString("$sb\n", mode: FileMode.append); + lastLine.lastChar = ch; + if (ch == _lineFeed) { + if (showEnds) { + sb.write('\$'); + } + } else if (ch == tab) { + if (showTabs) { + sb.write('^I'); + continue; + } + } else if (showNonPrinting) { + if (ch >= 32) { + if (ch < 127) { + // ASCII + sb.writeCharCode(ch); + continue; + } else if (ch == 127) { + // NULL + sb.write('^?'); + continue; + } else { + // UNICODE + sb.write('U+' + ch.toRadixString(16).padLeft(4, '0').toUpperCase()); + continue; + } + } else { + sb + ..write('^') + ..writeCharCode(ch + 64); + continue; + } } - } catch (e) { - rethrow; + sb.writeCharCode(ch); } - } - return lineNumber; -} + if (sb.isNotEmpty) { + out.write(sb); + } + }); +} \ No newline at end of file diff --git a/pubspec.lock b/pubspec.lock index 84b7283..e310a66 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -274,6 +274,13 @@ packages: url: "https://pub.dartlang.org" source: hosted version: "1.1.0" + string_splitter: + dependency: "direct main" + description: + name: string_splitter + url: "https://pub.dartlang.org" + source: hosted + version: "1.0.0+1" term_glyph: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index fa4356c..07cde21 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -12,3 +12,4 @@ dev_dependencies: dependencies: args: ^2.3.0 indent: ^2.0.0 + string_splitter: ^1.0.0+1 diff --git a/test/dcat_test.dart b/test/dcat_test.dart index 205af4a..174a090 100644 --- a/test/dcat_test.dart +++ b/test/dcat_test.dart @@ -10,16 +10,16 @@ import 'package:test/test.dart'; import '../bin/dcat.dart' as app; void main() { + const sampleBinary = 'test/test.7z'; + const sampleFile = 'test/test.txt'; + const sampleText = 'This is a test'; + const sourceFile = 'bin/dcat.dart'; + int exitCode; - final List log = []; - final sampleBinary = 'test/test.7z'; - final sampleFile = 'test/test.txt'; - final sampleText = 'This is a test'; - final sourceFile = 'bin/dcat.dart'; final tempDir = Directory.systemTemp.createTempSync(); - Stream> mockStdin() async* { - yield sampleText.codeUnits; + Stream> mockStdin({String text = sampleText}) async* { + yield text.codeUnits; } File tmpFile() => @@ -66,64 +66,83 @@ void main() { }); group('lib', () { + test('Test CatResult', () async { + final result = CatResult(); + expect(result.isSuccess, true, reason: 'success by default'); + result.addMessage(exitFailure, sampleText); + expect(result.isFailure, true, reason: 'is failure'); + expect(result.messages.first, equals(sampleText), reason: 'message is sample'); + }); + test('Test cat source', () async { - await cat([sourceFile], stdout, log: log); - expect(log.isEmpty, false, reason: 'log is empty'); - expect(log.first, startsWith('// Copyright (c)'), + final tmp = tmpFile(); + await cat([sourceFile], tmp.openWrite()); + final lines = await tmp.readAsLines(); + expect(lines.isEmpty, false, reason: 'log is empty'); + expect(lines.first, startsWith('// Copyright (c)'), reason: 'has copyright'); - expect(log.last, equals('}')); + expect(lines.last, equals('}')); }); test('Test cat -n source', () async { + final tmp = tmpFile(); final result = - await cat([sourceFile], stdout, log: log, showLineNumbers: true); + await cat([sourceFile], tmp.openWrite(), showLineNumbers: true); expect(result.exitCode, 0, reason: 'result code is 0'); - expect(log.first, startsWith(' 1 // Copyright (c)'), + final lines = await tmp.readAsLines(); + expect(lines.first, startsWith(' 1\t// Copyright (c)'), reason: 'has copyright'); - expect(log.last, endsWith(' }'), reason: 'last line'); - for (final String line in log) { - expect(line, matches('^ +\\d+ .*'), reason: 'has line number'); + expect(lines.last, endsWith('\t}'), reason: 'last line'); + for (final line in lines) { + expect(line, matches('^ +\\d+\t.*'), reason: 'has line number'); } }); test('Test cat source test', () async { - await cat([sourceFile, sampleFile], stdout, log: log); - expect(log.length, greaterThan(10), reason: 'more than 10 lines'); - expect(log.first, startsWith('// Copyright'), + final tmp = tmpFile(); + await cat([sourceFile, sampleFile], tmp.openWrite()); + final lines = await tmp.readAsLines(); + expect(lines.length, greaterThan(10), reason: 'more than 10 lines'); + expect(lines.first, startsWith('// Copyright'), reason: 'start with copyright'); - expect(log.last, endsWith('✓'), reason: 'end with checkmark'); + expect(lines.last, endsWith('✓'), reason: 'end with checkmark'); }); test('Test cat -E', () async { - await cat([sampleFile], stdout, log: log, showEnds: true); + final tmp = tmpFile(); + await cat([sampleFile], tmp.openWrite(), showEnds: true); var hasBlank = false; - for (final String line in log) { - expect(line, endsWith('\$')); - if (line == '\$') { + final lines = await tmp.readAsLines(); + for (var i = 0; i < lines.length - 1; i++) { + expect(lines[i], endsWith('\$')); + if (lines[i] == '\$') { hasBlank = true; } } expect(hasBlank, true, reason: 'has blank line'); - expect(log.last, endsWith('✓\$'), reason: 'has unicode'); + expect(lines.last, endsWith('✓'), reason: 'has unicode'); }); test('Test cat -bE', () async { - await cat([sampleFile], stdout, - log: log, numberNonBlank: true, showEnds: true); - var hasBlank = false; - for (final String line in log) { - expect(line, endsWith('\$')); - if (line.contains(RegExp(r'^ +\d+ .*\$$'))) { - hasBlank = true; + final tmp = tmpFile(); + await cat([sampleFile], tmp.openWrite(), + numberNonBlank: true, showEnds: true); + final lines = await tmp.readAsLines(); + for (var i = 0; i < lines.length - 1; i++) { + expect(lines[i], endsWith('\$'), reason: '${lines[i]} ends with \$'); + if (lines[i] != '\$') { + expect(lines[i], contains(RegExp(r'^ +\d+\t.*\$$')), + reason: '${lines[i]} is valid'); } } - expect(hasBlank, true, reason: 'has blank line'); }); test('Test cat -T', () async { - await cat([sampleFile], stdout, log: log, showTabs: true); + final tmp = tmpFile(); + await cat([sampleFile], tmp.openWrite(), showTabs: true); var hasTab = false; - for (final String line in log) { + final lines = await tmp.readAsLines(); + for (final String line in lines) { if (line.startsWith('^I')) { hasTab = true; break; @@ -133,10 +152,12 @@ void main() { }); test('Test cat -s', () async { - await cat([sampleFile], stdout, log: log, squeezeBlank: true); + final tmp = tmpFile(); + await cat([sampleFile], tmp.openWrite(), squeezeBlank: true); var hasSqueeze = true; var prevLine = 'foo'; - for (final String line in log) { + final lines = await tmp.readAsLines(); + for (final String line in lines) { if (line == prevLine) { hasSqueeze = false; } @@ -146,27 +167,34 @@ void main() { }); test('Test cat -A', () async { - await cat([sampleFile], stdout, - log: log, showNonPrinting: true, showEnds: true, showTabs: true); - expect(log.last, equals('^I^A^B^C^DU+00A9^?U+0080U+2713\$')); + final tmp = tmpFile(); + await cat([sampleFile], tmp.openWrite(), + showNonPrinting: true, showEnds: true, showTabs: true); + final lines = await tmp.readAsLines(); + expect(lines.first, endsWith('\$'), reason: '\$ at end.'); + expect(lines.last, equals('^I^A^B^C^DU+00A9^?U+0080U+2713'), + reason: "no last linefeed"); }); test('Test cat -t', () async { - await cat([sampleFile], stdout, - log: log, showNonPrinting: true, showTabs: true); - expect(log.last, equals('^I^A^B^C^DU+00A9^?U+0080U+2713')); + final tmp = tmpFile(); + await cat([sampleFile], tmp.openWrite(), + showNonPrinting: true, showTabs: true); + final lines = await tmp.readAsLines(); + expect(lines.last, equals('^I^A^B^C^DU+00A9^?U+0080U+2713')); }); - test('Test cat-Abs', () async { - await cat([sampleFile], stdout, - log: log, + test('Test cat -Abs', () async { + final tmp = tmpFile(); + await cat([sampleFile], tmp.openWrite(), showNonPrinting: true, showEnds: true, showTabs: true, numberNonBlank: true, squeezeBlank: true); var blankLines = 0; - for (final String line in log) { + final lines = await tmp.readAsLines(); + for (final String line in lines) { if (line == '\$') { blankLines++; } @@ -175,23 +203,25 @@ void main() { }); test('Test cat -v', () async { - await cat([sampleFile], stdout, log: log, showNonPrinting: true); + final tmp = tmpFile(); + await cat([sampleFile], tmp.openWrite(), showNonPrinting: true); var hasTab = false; - for (final String line in log) { + final lines = await tmp.readAsLines(); + for (final String line in lines) { if (line.contains('\t')) { hasTab = true; break; } } expect(hasTab, true, reason: "has real tab"); - expect(log.last, equals('\t^A^B^C^DU+00A9^?U+0080U+2713'), + expect(lines.last, equals('\t^A^B^C^DU+00A9^?U+0080U+2713'), reason: 'non-printing'); }); test('Test cat to file', () async { final tmp = tmpFile(); - final result = await cat([sampleFile], tmp, log: log); - expect(result.exitCode, exitSuccess, reason: 'result code is success'); + final result = await cat([sampleFile], tmp.openWrite()); + expect(result.isSuccess, true, reason: 'result code is success'); expect(result.messages.length, 0, reason: 'messages is empty'); expect(await tmp.exists(), true, reason: 'tmp file exists'); expect(await tmp.length(), greaterThan(0), @@ -202,9 +232,8 @@ void main() { }); test('Test cat with file and binary', () async { - final tmp = tmpFile(); - final result = await cat([sampleFile, sampleBinary], tmp, log: log); - expect(result.exitCode, exitFailure, reason: 'result code is failure'); + final result = await cat([sampleFile, sampleBinary], stdout); + expect(result.isFailure, true, reason: 'result code is failure'); expect(result.messages.length, 1, reason: 'as one message'); expect(result.messages.first, contains('Binary'), reason: 'message contains binary'); @@ -212,25 +241,33 @@ void main() { test('Test empty stdin', () async { final tmp = tmpFile(); - var result = await cat([], tmp, input: Stream.empty()); + var result = await cat([], tmp.openWrite(), input: Stream.empty()); expect(result.exitCode, exitSuccess, reason: 'cat() is successful'); expect(result.messages.length, 0, reason: 'cat() has no message'); - result = await cat(['-'], tmp, input: Stream.empty()); + result = await cat(['-'], tmp.openWrite(), input: Stream.empty()); expect(result.exitCode, exitSuccess, reason: 'cat(-) is successful'); expect(result.messages.length, 0, reason: 'cat(-) no message'); }); - test('Test cat with stdin', () async { + test('Test cat -', () async { var tmp = tmpFile(); - var result = await cat(['-'], tmp, input: mockStdin()); - expect(result.exitCode, exitSuccess, reason: 'result code is failure'); + final result = await cat(['-'], tmp.openWrite(), input: mockStdin()); + expect(result.exitCode, exitSuccess, reason: 'result code is successful'); expect(result.messages.length, 0, reason: 'no message'); - var lines = await tmp.readAsLines(); tmp = tmpFile(); expect(await tmp.exists(), false, reason: 'tmp file does not exists'); - result = await cat([], tmp, input: mockStdin()); + }); + + test('Test cat()', () async { + var tmp = tmpFile(); + await cat([], tmp.openWrite(), input: mockStdin()); + var lines = await tmp.readAsLines(); expect(lines.first, equals(sampleText), reason: 'cat() is sample text'); + tmp = tmpFile(); + await cat([], tmp.openWrite(), input: mockStdin(text: "Line 1\nLine 2")); + lines = await tmp.readAsLines(); + expect(lines.length, 2, reason: "two lines"); }); }); }