Skip to content

Add support for Python t-strings #328

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jpe wants to merge 1 commit into from
Closed

Add support for Python t-strings #328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 15 additions & 5 deletions lexers/LexPython.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ namespace {

The PEP for f-strings is at https://www.python.org/dev/peps/pep-0498/
*/
/* t-string are lexed identically to f-strings and use the stack of state values
and the lexer states for f-strings
*/
struct SingleFStringExpState {
int state;
int nestingCount;
Expand All @@ -62,7 +65,7 @@ struct SingleFStringExpState {
/* kwCDef, kwCTypeName only used for Cython */
enum kwType { kwOther, kwClass, kwDef, kwImport, kwCDef, kwCTypeName, kwCPDef };

enum literalsAllowed { litNone = 0, litU = 1, litB = 2, litF = 4 };
enum literalsAllowed { litNone = 0, litU = 1, litB = 2, litF = 4, litT = 8 };

constexpr int indicatorWhitespace = 1;

Expand All @@ -74,7 +77,8 @@ constexpr bool IsPyStringTypeChar(int ch, literalsAllowed allowed) noexcept {
return
((allowed & litB) && (ch == 'b' || ch == 'B')) ||
((allowed & litU) && (ch == 'u' || ch == 'U')) ||
((allowed & litF) && (ch == 'f' || ch == 'F'));
((allowed & litF) && (ch == 'f' || ch == 'F')) ||
((allowed & litT) && (ch == 't' || ch == 'T'));
}

constexpr bool IsQuote(int ch) {
Expand All @@ -88,7 +92,7 @@ constexpr bool IsRawPrefix(int ch) {
bool IsPyStringStart(int ch, int chNext, int chNext2, literalsAllowed allowed) noexcept {
// To cover both python2 and python3 lex character prefixes as --
// ur'' is a string, but ru'' is not
// fr'', rf'', br'', rb'' are all strings
// fr'', rf'', br'', rb'', tr'', rt'' are all strings
if (IsQuote(ch))
return true;
if (IsPyStringTypeChar(ch, allowed)) {
Expand Down Expand Up @@ -163,14 +167,14 @@ int GetPyStringState(Accessor &styler, Sci_Position i, Sci_PositionU *nextIndex,
if (IsRawPrefix(ch)) {
i++;
if (IsPyStringTypeChar(chNext, allowed)) {
if (AnyOf(chNext, 'f', 'F'))
if (AnyOf(chNext, 'f', 'F', 't', 'T'))
isFString = true;
i++;
}
ch = styler.SafeGetCharAt(i);
chNext = styler.SafeGetCharAt(i + 1);
} else if (IsPyStringTypeChar(ch, allowed)) {
if (AnyOf(ch, 'f', 'F'))
if (AnyOf(ch, 'f', 'F', 't', 'T'))
isFString = true;
if (IsRawPrefix(chNext))
i += 2;
Expand Down Expand Up @@ -291,6 +295,7 @@ struct OptionsPython {
bool stringsU = true;
bool stringsB = true;
bool stringsF = true;
bool stringsT = true;
bool stringsOverNewline = false;
bool keywords2NoSubIdentifiers = false;
bool fold = false;
Expand All @@ -307,6 +312,8 @@ struct OptionsPython {
allowedLiterals = static_cast<literalsAllowed>(allowedLiterals | litB);
if (stringsF)
allowedLiterals = static_cast<literalsAllowed>(allowedLiterals | litF);
if (stringsT)
allowedLiterals = static_cast<literalsAllowed>(allowedLiterals | litT);
return allowedLiterals;
}
};
Expand Down Expand Up @@ -343,6 +350,9 @@ struct OptionSetPython : public OptionSet<OptionsPython> {
DefineProperty("lexer.python.strings.f.pep.701", &OptionsPython::pep701StringsF,
"Set to 0 to use pre-PEP 701 / Python 3.12 f-string lexing.");

DefineProperty("lexer.python.strings.t", &OptionsPython::stringsT,
"Set to 0 to not recognise Python 3.14 t-string literals f\"var={var}\".");
Copy link
Copy Markdown
Contributor

@zufuliu zufuliu Aug 23, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

A typo, t\"var={var}\"


DefineProperty("lexer.python.strings.over.newline", &OptionsPython::stringsOverNewline,
"Set to 1 to allow strings to span newline characters.");

Expand Down
16 changes: 16 additions & 0 deletions test/examples/python/t-strings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Simple nesting
t" { "" } "

# Multi-line field with comment
t" {

"" # comment

} "

# Single quoted continued with \
t" \
"

# 4 nested t-strings
t'Outer {t"nested {1} {t"nested {2} {t"nested {3} {t"nested {4}"}"}"}"}'
17 changes: 17 additions & 0 deletions test/examples/python/t-strings.py.folded
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
0 400 0 # Simple nesting
0 400 0 t" { "" } "
1 400 0
0 400 0 # Multi-line field with comment
0 400 0 t" {
1 400 0
0 400 0 "" # comment
1 400 0
0 400 0 } "
1 400 0
0 400 0 # Single quoted continued with \
0 400 0 t" \
0 400 0 "
1 400 0
0 400 0 # 4 nested t-strings
0 400 0 t'Outer {t"nested {1} {t"nested {2} {t"nested {3} {t"nested {4}"}"}"}"}'
1 400 0
16 changes: 16 additions & 0 deletions test/examples/python/t-strings.py.styled
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{1}# Simple nesting{0}
{16}t" {{0} {3}""{0} {16}} "{0}

{1}# Multi-line field with comment{0}
{16}t" {{0}

{3}""{0} {1}# comment{0}

{16}} "{0}

{1}# Single quoted continued with \{0}
{16}t" \
"{0}

{1}# 4 nested t-strings{0}
{17}t'Outer {{16}t"nested {{2}1{16}} {t"nested {{2}2{16}} {t"nested {{2}3{16}} {t"nested {{2}4{16}}"}"}"}"{17}}'{0}