Rewrote data parsing using streams.

This commit is contained in:
Erik C. Thauvin 2021-10-16 00:54:06 -07:00
parent 42afc67d46
commit 4ee7ec2127
6 changed files with 273 additions and 223 deletions

2
.gitignore vendored
View file

@ -51,7 +51,6 @@
/**/.idea/**/workspace.xml
/**/.idea/sonarlint*
/**/.idea_modules/
/coverage/
Thumbs.db
__pycache__
atlassian-ide-plugin.xml
@ -60,6 +59,7 @@ bin/dcat.exe
build/
cmake-build-*/
com_crashlytics_export_strings.xml
coverage/
crashlytics-build.properties
crashlytics.properties
dependency-reduced-pom.xml

View file

@ -11,8 +11,8 @@ import 'package:indent/indent.dart';
const appName = 'dcat';
const appVersion = '1.0.0';
const helpFlag = 'help';
const nonBlankFlag = 'number-nonblank';
const numberFlag = 'number';
const numberNonBlank = 'number-nonblank';
const showAllFlag = 'show-all';
const showEndsFlag = 'show-ends';
const showNonPrintingEndsFlag = 'show-nonprinting-ends';
@ -26,13 +26,71 @@ const versionFlag = 'version';
///
/// Usage: `dcat [option] [file]`
Future<int> main(List<String> arguments) async {
final parser = ArgParser();
exitCode = exitSuccess;
final parser = await setupArgsParser();
final ArgResults argResults;
try {
argResults = parser.parse(arguments);
} on FormatException catch (e) {
return printError(
"${e.message}\nTry '$appName --$helpFlag' for more information.");
}
if (argResults[helpFlag]) {
exitCode = await usage(parser.usage);
} else if (argResults[versionFlag]) {
exitCode = await printVersion();
} else {
final paths = argResults.rest;
var showEnds = argResults[showEndsFlag];
var showTabs = argResults[showTabsFlag];
var showLineNumbers = argResults[numberFlag];
var showNonBlank = argResults[numberNonBlank];
var showNonPrinting = argResults[showNonPrintingFlag];
if (argResults[showNonPrintingEndsFlag]) {
showNonPrinting = showEnds = true;
}
if (argResults[showNonPrintingTabsFlag]) {
showNonPrinting = showTabs = true;
}
if (argResults[showAllFlag]) {
showNonPrinting = showEnds = showTabs = true;
}
if (showNonBlank) {
showLineNumbers = true;
}
final result = await cat(paths, stdout,
input: stdin,
showEnds: showEnds,
showLineNumbers: showLineNumbers,
numberNonBlank: showNonBlank,
showTabs: showTabs,
squeezeBlank: argResults[squeezeBlank],
showNonPrinting: showNonPrinting);
for (final message in result.messages) {
await printError(message);
}
exitCode = result.exitCode;
}
return exitCode;
}
/// Setup the command-line arguments parser.
Future<ArgParser> setupArgsParser() async {
final parser = ArgParser();
parser.addFlag(showAllFlag,
negatable: false, abbr: 'A', help: 'equivalent to -vET');
parser.addFlag(nonBlankFlag,
parser.addFlag(numberNonBlank,
negatable: false,
abbr: 'b',
help: 'number nonempty output lines, overrides -n');
@ -60,53 +118,7 @@ Future<int> main(List<String> arguments) async {
abbr: 'v',
help: 'use ^ and U+ notation, except for LFD and TAB');
final ArgResults argResults;
try {
argResults = parser.parse(arguments);
} on FormatException catch (e) {
return printError(
"${e.message}\nTry '$appName --$helpFlag' for more information.");
}
if (argResults[helpFlag]) {
exitCode = await usage(parser.usage);
} else if (argResults[versionFlag]) {
exitCode = await printVersion();
} else {
final paths = argResults.rest;
var showEnds = argResults[showEndsFlag];
var showTabs = argResults[showTabsFlag];
var showNonPrinting = argResults[showNonPrintingFlag];
if (argResults[showNonPrintingEndsFlag]) {
showNonPrinting = showEnds = true;
}
if (argResults[showNonPrintingTabsFlag]) {
showNonPrinting = showTabs = true;
}
if (argResults[showAllFlag]) {
showNonPrinting = showEnds = showTabs = true;
}
final result = await cat(paths, stdout,
input: stdin,
showEnds: showEnds,
showLineNumbers: argResults[numberFlag],
numberNonBlank: argResults[nonBlankFlag],
showTabs: showTabs,
squeezeBlank: argResults[squeezeBlank],
showNonPrinting: showNonPrinting);
for (final message in result.messages) {
await printError(message);
}
exitCode = result.exitCode;
}
return exitCode;
return parser;
}
/// Prints the error [message] to [stderr].

View file

@ -8,19 +8,31 @@ library dcat;
import 'dart:convert';
import 'dart:io';
/// Failure exit code.
const exitFailure = 1;
/// Success exit code.
const exitSuccess = 0;
const _lineFeed = 10;
/// Holds the [cat] result [exitCode] and error [messages].
class CatResult {
/// The exit code.
int exitCode = exitSuccess;
/// The error messages.
final List<String> messages = [];
CatResult();
/// Add a message.
/// Returns `true` if the [exitCode] is [exitFailure].
bool get isFailure => exitCode == exitFailure;
/// Returns `true` if the [exitCode] is [exitSuccess].
bool get isSuccess => exitCode == exitSuccess;
/// Add a message with an optional path.
void addMessage(int exitCode, String message, {String? path}) {
this.exitCode = exitCode;
if (path != null && path.isNotEmpty) {
@ -31,40 +43,35 @@ class CatResult {
}
}
/// Concatenates files in [paths] to [stdout] or [File].
// Holds the current line number and last character.
class _LastLine {
int lineNumber;
int lastChar;
_LastLine(this.lineNumber, this.lastChar);
}
/// Concatenates files in [paths] to the standard output or a file.
///
/// * [output] should be an [IOSink] like [stdout] or a [File].
/// * [output] should be an [IOSink] such as [stdout] or [File.openWrite].
/// * [input] can be [stdin].
/// * [log] is used for debugging/testing purposes.
///
/// The remaining optional parameters are similar to the [GNU cat utility](https://www.gnu.org/software/coreutils/manual/html_node/cat-invocation.html#cat-invocation).
Future<CatResult> cat(List<String> paths, Object output,
Future<CatResult> cat(List<String> paths, IOSink output,
{Stream<List<int>>? input,
List<String>? log,
bool showEnds = false,
bool numberNonBlank = false,
bool showLineNumbers = false,
bool showTabs = false,
bool squeezeBlank = false,
bool showNonPrinting = false}) async {
var result = CatResult();
var lineNumber = 1;
log?.clear();
final result = CatResult();
final lastLine = _LastLine(0, _lineFeed);
if (paths.isEmpty) {
if (input != null) {
final lines = await _readStream(input);
try {
await _writeLines(
lines,
lineNumber,
output,
log,
showEnds,
showLineNumbers,
numberNonBlank,
showTabs,
squeezeBlank,
showNonPrinting);
await _writeStream(input, lastLine, output, showEnds, showLineNumbers,
numberNonBlank, showTabs, squeezeBlank, showNonPrinting);
} catch (e) {
result.addMessage(exitFailure, '$e');
}
@ -72,25 +79,14 @@ Future<CatResult> cat(List<String> paths, Object output,
} else {
for (final path in paths) {
try {
final Stream<String> lines;
final Stream<List<int>> stream;
if (path == '-' && input != null) {
lines = await _readStream(input);
stream = input;
} else {
lines = utf8.decoder
.bind(File(path).openRead())
.transform(const LineSplitter());
stream = File(path).openRead();
}
lineNumber = await _writeLines(
lines,
lineNumber,
output,
log,
showEnds,
showLineNumbers,
numberNonBlank,
showTabs,
squeezeBlank,
showNonPrinting);
await _writeStream(stream, lastLine, output, showEnds, showLineNumbers,
numberNonBlank, showTabs, squeezeBlank, showNonPrinting);
} on FileSystemException catch (e) {
final String? osMessage = e.osError?.message;
final String message;
@ -111,80 +107,77 @@ Future<CatResult> cat(List<String> paths, Object output,
return result;
}
/// Parses line with non-printing characters.
Future<String> _parseNonPrinting(String line, bool showTabs) async {
// Writes parsed data from a stream
Future<void> _writeStream(
Stream stream,
_LastLine lastLine,
IOSink out,
bool showEnds,
bool showLineNumbers,
bool numberNonBlank,
bool showTabs,
bool squeezeBlank,
bool showNonPrinting) async {
const tab = 9;
int squeeze = 0;
final sb = StringBuffer();
for (var ch in line.runes) {
if (ch >= 32) {
if (ch < 127) {
sb.writeCharCode(ch);
} else if (ch == 127) {
sb.write('^?');
} else {
sb.write('U+' + ch.toRadixString(16).padLeft(4, '0').toUpperCase());
}
} else if (ch == 9 && !showTabs) {
sb.write('\t');
} else {
sb
..write('^')
..writeCharCode(ch + 64);
}
}
return sb.toString();
}
/// Reads from stream (stdin, etc.)
Future<Stream<String>> _readStream(Stream<List<int>> input) async =>
input.transform(utf8.decoder).transform(const LineSplitter());
/// Writes lines to stdout.
Future<int> _writeLines(Stream<String> lines, int lineNumber, Object out,
[List<String>? log,
bool showEnds = false,
bool showLineNumbers = false,
bool showNonBlank = false,
bool showTabs = false,
bool squeezeBlank = false,
bool showNonPrinting = false]) async {
var emptyLine = 0;
final sb = StringBuffer();
await for (final line in lines) {
await stream.forEach((data) {
sb.clear();
if (squeezeBlank && line.isEmpty) {
if (++emptyLine >= 2) {
continue;
for (final ch in utf8.decode(data).runes) {
if (lastLine.lastChar == _lineFeed) {
if (squeezeBlank) {
if (ch == _lineFeed) {
if (squeeze >= 1) {
lastLine.lastChar = ch;
continue;
}
squeeze++;
} else {
squeeze = 0;
}
}
if (showLineNumbers || numberNonBlank) {
if (!numberNonBlank || ch != _lineFeed) {
sb.write('${++lastLine.lineNumber}'.padLeft(6) + '\t');
}
}
}
} else {
emptyLine = 0;
}
if (showLineNumbers || (showNonBlank && line.isNotEmpty)) {
sb.write('${lineNumber++} '.padLeft(8));
}
if (showNonPrinting) {
sb.write(await _parseNonPrinting(line, showTabs));
} else if (showTabs) {
sb.write(line.replaceAll('\t', '^I'));
} else {
sb.write(line);
}
if (showEnds) {
sb.write('\$');
}
log?.add(sb.toString());
try {
if (out is IOSink) {
out.writeln(sb);
} else if (out is File) {
await out.writeAsString("$sb\n", mode: FileMode.append);
lastLine.lastChar = ch;
if (ch == _lineFeed) {
if (showEnds) {
sb.write('\$');
}
} else if (ch == tab) {
if (showTabs) {
sb.write('^I');
continue;
}
} else if (showNonPrinting) {
if (ch >= 32) {
if (ch < 127) {
// ASCII
sb.writeCharCode(ch);
continue;
} else if (ch == 127) {
// NULL
sb.write('^?');
continue;
} else {
// UNICODE
sb.write('U+' + ch.toRadixString(16).padLeft(4, '0').toUpperCase());
continue;
}
} else {
sb
..write('^')
..writeCharCode(ch + 64);
continue;
}
}
} catch (e) {
rethrow;
sb.writeCharCode(ch);
}
}
return lineNumber;
}
if (sb.isNotEmpty) {
out.write(sb);
}
});
}

View file

@ -274,6 +274,13 @@ packages:
url: "https://pub.dartlang.org"
source: hosted
version: "1.1.0"
string_splitter:
dependency: "direct main"
description:
name: string_splitter
url: "https://pub.dartlang.org"
source: hosted
version: "1.0.0+1"
term_glyph:
dependency: transitive
description:

View file

@ -12,3 +12,4 @@ dev_dependencies:
dependencies:
args: ^2.3.0
indent: ^2.0.0
string_splitter: ^1.0.0+1

View file

@ -10,16 +10,16 @@ import 'package:test/test.dart';
import '../bin/dcat.dart' as app;
void main() {
const sampleBinary = 'test/test.7z';
const sampleFile = 'test/test.txt';
const sampleText = 'This is a test';
const sourceFile = 'bin/dcat.dart';
int exitCode;
final List<String> log = [];
final sampleBinary = 'test/test.7z';
final sampleFile = 'test/test.txt';
final sampleText = 'This is a test';
final sourceFile = 'bin/dcat.dart';
final tempDir = Directory.systemTemp.createTempSync();
Stream<List<int>> mockStdin() async* {
yield sampleText.codeUnits;
Stream<List<int>> mockStdin({String text = sampleText}) async* {
yield text.codeUnits;
}
File tmpFile() =>
@ -66,64 +66,83 @@ void main() {
});
group('lib', () {
test('Test CatResult', () async {
final result = CatResult();
expect(result.isSuccess, true, reason: 'success by default');
result.addMessage(exitFailure, sampleText);
expect(result.isFailure, true, reason: 'is failure');
expect(result.messages.first, equals(sampleText), reason: 'message is sample');
});
test('Test cat source', () async {
await cat([sourceFile], stdout, log: log);
expect(log.isEmpty, false, reason: 'log is empty');
expect(log.first, startsWith('// Copyright (c)'),
final tmp = tmpFile();
await cat([sourceFile], tmp.openWrite());
final lines = await tmp.readAsLines();
expect(lines.isEmpty, false, reason: 'log is empty');
expect(lines.first, startsWith('// Copyright (c)'),
reason: 'has copyright');
expect(log.last, equals('}'));
expect(lines.last, equals('}'));
});
test('Test cat -n source', () async {
final tmp = tmpFile();
final result =
await cat([sourceFile], stdout, log: log, showLineNumbers: true);
await cat([sourceFile], tmp.openWrite(), showLineNumbers: true);
expect(result.exitCode, 0, reason: 'result code is 0');
expect(log.first, startsWith(' 1 // Copyright (c)'),
final lines = await tmp.readAsLines();
expect(lines.first, startsWith(' 1\t// Copyright (c)'),
reason: 'has copyright');
expect(log.last, endsWith(' }'), reason: 'last line');
for (final String line in log) {
expect(line, matches('^ +\\d+ .*'), reason: 'has line number');
expect(lines.last, endsWith('\t}'), reason: 'last line');
for (final line in lines) {
expect(line, matches('^ +\\d+\t.*'), reason: 'has line number');
}
});
test('Test cat source test', () async {
await cat([sourceFile, sampleFile], stdout, log: log);
expect(log.length, greaterThan(10), reason: 'more than 10 lines');
expect(log.first, startsWith('// Copyright'),
final tmp = tmpFile();
await cat([sourceFile, sampleFile], tmp.openWrite());
final lines = await tmp.readAsLines();
expect(lines.length, greaterThan(10), reason: 'more than 10 lines');
expect(lines.first, startsWith('// Copyright'),
reason: 'start with copyright');
expect(log.last, endsWith(''), reason: 'end with checkmark');
expect(lines.last, endsWith(''), reason: 'end with checkmark');
});
test('Test cat -E', () async {
await cat([sampleFile], stdout, log: log, showEnds: true);
final tmp = tmpFile();
await cat([sampleFile], tmp.openWrite(), showEnds: true);
var hasBlank = false;
for (final String line in log) {
expect(line, endsWith('\$'));
if (line == '\$') {
final lines = await tmp.readAsLines();
for (var i = 0; i < lines.length - 1; i++) {
expect(lines[i], endsWith('\$'));
if (lines[i] == '\$') {
hasBlank = true;
}
}
expect(hasBlank, true, reason: 'has blank line');
expect(log.last, endsWith('\$'), reason: 'has unicode');
expect(lines.last, endsWith(''), reason: 'has unicode');
});
test('Test cat -bE', () async {
await cat([sampleFile], stdout,
log: log, numberNonBlank: true, showEnds: true);
var hasBlank = false;
for (final String line in log) {
expect(line, endsWith('\$'));
if (line.contains(RegExp(r'^ +\d+ .*\$$'))) {
hasBlank = true;
final tmp = tmpFile();
await cat([sampleFile], tmp.openWrite(),
numberNonBlank: true, showEnds: true);
final lines = await tmp.readAsLines();
for (var i = 0; i < lines.length - 1; i++) {
expect(lines[i], endsWith('\$'), reason: '${lines[i]} ends with \$');
if (lines[i] != '\$') {
expect(lines[i], contains(RegExp(r'^ +\d+\t.*\$$')),
reason: '${lines[i]} is valid');
}
}
expect(hasBlank, true, reason: 'has blank line');
});
test('Test cat -T', () async {
await cat([sampleFile], stdout, log: log, showTabs: true);
final tmp = tmpFile();
await cat([sampleFile], tmp.openWrite(), showTabs: true);
var hasTab = false;
for (final String line in log) {
final lines = await tmp.readAsLines();
for (final String line in lines) {
if (line.startsWith('^I')) {
hasTab = true;
break;
@ -133,10 +152,12 @@ void main() {
});
test('Test cat -s', () async {
await cat([sampleFile], stdout, log: log, squeezeBlank: true);
final tmp = tmpFile();
await cat([sampleFile], tmp.openWrite(), squeezeBlank: true);
var hasSqueeze = true;
var prevLine = 'foo';
for (final String line in log) {
final lines = await tmp.readAsLines();
for (final String line in lines) {
if (line == prevLine) {
hasSqueeze = false;
}
@ -146,27 +167,34 @@ void main() {
});
test('Test cat -A', () async {
await cat([sampleFile], stdout,
log: log, showNonPrinting: true, showEnds: true, showTabs: true);
expect(log.last, equals('^I^A^B^C^DU+00A9^?U+0080U+2713\$'));
final tmp = tmpFile();
await cat([sampleFile], tmp.openWrite(),
showNonPrinting: true, showEnds: true, showTabs: true);
final lines = await tmp.readAsLines();
expect(lines.first, endsWith('\$'), reason: '\$ at end.');
expect(lines.last, equals('^I^A^B^C^DU+00A9^?U+0080U+2713'),
reason: "no last linefeed");
});
test('Test cat -t', () async {
await cat([sampleFile], stdout,
log: log, showNonPrinting: true, showTabs: true);
expect(log.last, equals('^I^A^B^C^DU+00A9^?U+0080U+2713'));
final tmp = tmpFile();
await cat([sampleFile], tmp.openWrite(),
showNonPrinting: true, showTabs: true);
final lines = await tmp.readAsLines();
expect(lines.last, equals('^I^A^B^C^DU+00A9^?U+0080U+2713'));
});
test('Test cat-Abs', () async {
await cat([sampleFile], stdout,
log: log,
test('Test cat -Abs', () async {
final tmp = tmpFile();
await cat([sampleFile], tmp.openWrite(),
showNonPrinting: true,
showEnds: true,
showTabs: true,
numberNonBlank: true,
squeezeBlank: true);
var blankLines = 0;
for (final String line in log) {
final lines = await tmp.readAsLines();
for (final String line in lines) {
if (line == '\$') {
blankLines++;
}
@ -175,23 +203,25 @@ void main() {
});
test('Test cat -v', () async {
await cat([sampleFile], stdout, log: log, showNonPrinting: true);
final tmp = tmpFile();
await cat([sampleFile], tmp.openWrite(), showNonPrinting: true);
var hasTab = false;
for (final String line in log) {
final lines = await tmp.readAsLines();
for (final String line in lines) {
if (line.contains('\t')) {
hasTab = true;
break;
}
}
expect(hasTab, true, reason: "has real tab");
expect(log.last, equals('\t^A^B^C^DU+00A9^?U+0080U+2713'),
expect(lines.last, equals('\t^A^B^C^DU+00A9^?U+0080U+2713'),
reason: 'non-printing');
});
test('Test cat to file', () async {
final tmp = tmpFile();
final result = await cat([sampleFile], tmp, log: log);
expect(result.exitCode, exitSuccess, reason: 'result code is success');
final result = await cat([sampleFile], tmp.openWrite());
expect(result.isSuccess, true, reason: 'result code is success');
expect(result.messages.length, 0, reason: 'messages is empty');
expect(await tmp.exists(), true, reason: 'tmp file exists');
expect(await tmp.length(), greaterThan(0),
@ -202,9 +232,8 @@ void main() {
});
test('Test cat with file and binary', () async {
final tmp = tmpFile();
final result = await cat([sampleFile, sampleBinary], tmp, log: log);
expect(result.exitCode, exitFailure, reason: 'result code is failure');
final result = await cat([sampleFile, sampleBinary], stdout);
expect(result.isFailure, true, reason: 'result code is failure');
expect(result.messages.length, 1, reason: 'as one message');
expect(result.messages.first, contains('Binary'),
reason: 'message contains binary');
@ -212,25 +241,33 @@ void main() {
test('Test empty stdin', () async {
final tmp = tmpFile();
var result = await cat([], tmp, input: Stream.empty());
var result = await cat([], tmp.openWrite(), input: Stream.empty());
expect(result.exitCode, exitSuccess, reason: 'cat() is successful');
expect(result.messages.length, 0, reason: 'cat() has no message');
result = await cat(['-'], tmp, input: Stream.empty());
result = await cat(['-'], tmp.openWrite(), input: Stream.empty());
expect(result.exitCode, exitSuccess, reason: 'cat(-) is successful');
expect(result.messages.length, 0, reason: 'cat(-) no message');
});
test('Test cat with stdin', () async {
test('Test cat -', () async {
var tmp = tmpFile();
var result = await cat(['-'], tmp, input: mockStdin());
expect(result.exitCode, exitSuccess, reason: 'result code is failure');
final result = await cat(['-'], tmp.openWrite(), input: mockStdin());
expect(result.exitCode, exitSuccess, reason: 'result code is successful');
expect(result.messages.length, 0, reason: 'no message');
var lines = await tmp.readAsLines();
tmp = tmpFile();
expect(await tmp.exists(), false, reason: 'tmp file does not exists');
result = await cat([], tmp, input: mockStdin());
});
test('Test cat()', () async {
var tmp = tmpFile();
await cat([], tmp.openWrite(), input: mockStdin());
var lines = await tmp.readAsLines();
expect(lines.first, equals(sampleText), reason: 'cat() is sample text');
tmp = tmpFile();
await cat([], tmp.openWrite(), input: mockStdin(text: "Line 1\nLine 2"));
lines = await tmp.readAsLines();
expect(lines.length, 2, reason: "two lines");
});
});
}