From 2ca14700107febfdbe27c37d1ca6f88aa3f2d0cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Verschelde?= Date: Sun, 31 Jul 2022 18:46:53 +0200 Subject: [PATCH] File: Re-add support to skip CR (`\r`) in `File::get_as_text` This was removed in #63481, and we confirmed that it's better like this, but we add back the possibility to strip CR as an option, to optionally restore the previous behavior. For performance this is done directly in `String::parse_utf8`. Also fixes Android `FileAccess::get_line()` as this one _should_ strip CR. Supersedes #63717. (cherry picked from commit 1418f97c70a5551bdbfeea853cbc479b32ea9e08) --- core/bind/core_bind.cpp | 6 +++--- core/bind/core_bind.h | 2 +- core/os/file_access.cpp | 4 ++-- core/os/file_access.h | 2 +- core/ustring.cpp | 12 +++++++++++- core/ustring.h | 2 +- doc/classes/File.xml | 5 +++-- platform/android/file_access_filesystem_jandroid.cpp | 6 ++++-- 8 files changed, 26 insertions(+), 13 deletions(-) diff --git a/core/bind/core_bind.cpp b/core/bind/core_bind.cpp index 3ff3c0ee1..f83238bb8 100644 --- a/core/bind/core_bind.cpp +++ b/core/bind/core_bind.cpp @@ -2087,13 +2087,13 @@ PoolVector _File::get_buffer(int64_t p_length) const { return data; } -String _File::get_as_text() const { +String _File::get_as_text(bool p_skip_cr) const { ERR_FAIL_COND_V_MSG(!f, String(), "File must be opened before use."); uint64_t original_pos = f->get_position(); f->seek(0); - String text = f->get_as_utf8_string(); + String text = f->get_as_utf8_string(p_skip_cr); f->seek(original_pos); @@ -2285,7 +2285,7 @@ void _File::_bind_methods() { ClassDB::bind_method(D_METHOD("get_buffer", "len"), &_File::get_buffer); ClassDB::bind_method(D_METHOD("get_line"), &_File::get_line); ClassDB::bind_method(D_METHOD("get_csv_line", "delim"), &_File::get_csv_line, DEFVAL(",")); - ClassDB::bind_method(D_METHOD("get_as_text"), &_File::get_as_text); + ClassDB::bind_method(D_METHOD("get_as_text", "skip_cr"), &_File::get_as_text, DEFVAL(true)); ClassDB::bind_method(D_METHOD("get_md5", "path"), &_File::get_md5); ClassDB::bind_method(D_METHOD("get_sha256", "path"), &_File::get_sha256); ClassDB::bind_method(D_METHOD("get_endian_swap"), &_File::get_endian_swap); diff --git a/core/bind/core_bind.h b/core/bind/core_bind.h index 6dc9704af..a3595b3c4 100644 --- a/core/bind/core_bind.h +++ b/core/bind/core_bind.h @@ -549,7 +549,7 @@ public: PoolVector get_buffer(int64_t p_length) const; // Get an array of bytes. String get_line() const; Vector get_csv_line(const String &p_delim = ",") const; - String get_as_text() const; + String get_as_text(bool p_skip_cr = true) const; // Skip CR by default for compat. String get_md5(const String &p_path) const; String get_sha256(const String &p_path) const; diff --git a/core/os/file_access.cpp b/core/os/file_access.cpp index aca771c9d..709266dd2 100644 --- a/core/os/file_access.cpp +++ b/core/os/file_access.cpp @@ -378,7 +378,7 @@ uint64_t FileAccess::get_buffer(uint8_t *p_dst, uint64_t p_length) const { return i; } -String FileAccess::get_as_utf8_string() const { +String FileAccess::get_as_utf8_string(bool p_skip_cr) const { PoolVector sourcef; uint64_t len = get_len(); sourcef.resize(len + 1); @@ -389,7 +389,7 @@ String FileAccess::get_as_utf8_string() const { w[len] = 0; String s; - if (s.parse_utf8((const char *)w.ptr())) { + if (s.parse_utf8((const char *)w.ptr(), -1, p_skip_cr)) { return String(); } return s; diff --git a/core/os/file_access.h b/core/os/file_access.h index d226422c8..39bf6d8ca 100644 --- a/core/os/file_access.h +++ b/core/os/file_access.h @@ -113,7 +113,7 @@ public: virtual String get_line() const; virtual String get_token() const; virtual Vector get_csv_line(const String &p_delim = ",") const; - virtual String get_as_utf8_string() const; + virtual String get_as_utf8_string(bool p_skip_cr = true) const; // Skip CR by default for compat. /**< use this for files WRITTEN in _big_ endian machines (ie, amiga/mac) * It's not about the current CPU type but file formats. diff --git a/core/ustring.cpp b/core/ustring.cpp index 497570089..c003d41eb 100644 --- a/core/ustring.cpp +++ b/core/ustring.cpp @@ -1445,7 +1445,7 @@ String String::utf8(const char *p_utf8, int p_len) { return ret; }; -bool String::parse_utf8(const char *p_utf8, int p_len) { +bool String::parse_utf8(const char *p_utf8, int p_len, bool p_skip_cr) { #define _UNICERROR(m_err) print_line("Unicode error: " + String(m_err)); if (!p_utf8) { @@ -1477,6 +1477,11 @@ bool String::parse_utf8(const char *p_utf8, int p_len) { if (skip == 0) { uint8_t c = *ptrtmp >= 0 ? *ptrtmp : uint8_t(256 + *ptrtmp); + if (p_skip_cr && c == '\r') { + ptrtmp++; + continue; + } + /* Determine the number of characters in sequence */ if ((c & 0x80) == 0) { skip = 0; @@ -1534,6 +1539,11 @@ bool String::parse_utf8(const char *p_utf8, int p_len) { while (cstr_size) { int len = 0; + if (p_skip_cr && *p_utf8 == '\r') { + p_utf8++; + continue; + } + /* Determine the number of characters in sequence */ if ((*p_utf8 & 0x80) == 0) { len = 1; diff --git a/core/ustring.h b/core/ustring.h index 56eb1a64b..d244dda91 100644 --- a/core/ustring.h +++ b/core/ustring.h @@ -328,7 +328,7 @@ public: CharString ascii(bool p_allow_extended = false) const; CharString utf8() const; - bool parse_utf8(const char *p_utf8, int p_len = -1); //return true on error + bool parse_utf8(const char *p_utf8, int p_len = -1, bool p_skip_cr = false); //return true on error static String utf8(const char *p_utf8, int p_len = -1); static uint32_t hash(const CharType *p_cstr, int p_len); /* hash the string */ diff --git a/doc/classes/File.xml b/doc/classes/File.xml index 2d2a9d578..b6913db6e 100644 --- a/doc/classes/File.xml +++ b/doc/classes/File.xml @@ -87,9 +87,10 @@ + - Returns the whole file as a [String]. - Text is interpreted as being UTF-8 encoded. + Returns the whole file as a [String]. Text is interpreted as being UTF-8 encoded. + If [code]skip_cr[/code] is [code]true[/code], carriage return characters ([code]\r[/code], CR) will be ignored when parsing the UTF-8, so that only line feed characters ([code]\n[/code], LF) represent a new line (Unix convention). diff --git a/platform/android/file_access_filesystem_jandroid.cpp b/platform/android/file_access_filesystem_jandroid.cpp index 7149795ba..ec9ce759c 100644 --- a/platform/android/file_access_filesystem_jandroid.cpp +++ b/platform/android/file_access_filesystem_jandroid.cpp @@ -29,8 +29,10 @@ /*************************************************************************/ #include "file_access_filesystem_jandroid.h" + #include "core/os/os.h" #include "thread_jandroid.h" + #include jobject FileAccessFilesystemJAndroid::file_access_handler = nullptr; @@ -197,7 +199,7 @@ String FileAccessFilesystemJAndroid::get_line() const { if (elem == '\n' || elem == '\0') { // Found the end of the line const_cast(this)->seek(start_position + line_buffer_position + 1); - if (result.parse_utf8((const char *)line_buffer.ptr(), line_buffer_position)) { + if (result.parse_utf8((const char *)line_buffer.ptr(), line_buffer_position, true)) { return String(); } return result; @@ -205,7 +207,7 @@ String FileAccessFilesystemJAndroid::get_line() const { } } - if (result.parse_utf8((const char *)line_buffer.ptr(), line_buffer_position)) { + if (result.parse_utf8((const char *)line_buffer.ptr(), line_buffer_position, true)) { return String(); } return result;