First model version

Dockerfile

@@ -0,0 +1,29 @@
+FROM misakiminato/cuda-python:cu12.0.0-py3.8.16-devel-ubuntu18.04
+WORKDIR /app
+
+COPY ./requirements.txt /app/requirements.txt
+COPY ./packages.txt /app/packages.txt
+RUN pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html
+RUN apt-get update && xargs -r -a /app/packages.txt apt-get install -y && rm -rf /var/lib/apt/lists/*
+RUN pip3 install --no-cache-dir -r /app/requirements.txt
+
+
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+
+# Switch to the "user" user
+USER user
+
+# Set home to the user's home directory
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+
+EXPOSE 8501
+CMD streamlit run app.py --server.maxUploadSize 1024 --server.enableWebsocketCompression=false --server.enableXsrfProtection=false

Dockerfile.back

@@ -0,0 +1,35 @@
+FROM nvidia/cuda:12.0.0-base-ubuntu20.04
+ARG DEBIAN_FRONTEND=noninteractive
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+# Install python, git & ffmpeg
+RUN apt-get update && apt-get install --no-install-recommends -y \
+  build-essential \
+  python3.8=3.8.10* \
+  python3-pip \
+  git \
+  ffmpeg \
+  && apt-get clean && rm -rf /var/lib/apt/lists/*
+COPY ./requirements.txt /app/requirements.txt
+COPY ./packages.txt /app/packages.txt
+RUN pip install --upgrade pip
+RUN pip install pyproject-toml
+RUN pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html
+RUN apt-get update && xargs -r -a /app/packages.txt apt-get install -y && rm -rf /var/lib/apt/lists/*
+RUN pip3 install --no-cache-dir -r /app/requirements.txt
+RUN pip3 install --no-cache-dir numba==0.56.3
+RUN pip install --no-binary :all: pyworld
+
+WORKDIR /app
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+EXPOSE 8501
+CMD nvidia-smi -l
+CMD streamlit run app.py --server.maxUploadSize 1024 --server.enableWebsocketCompression=false --server.enableXsrfProtection=false

Dockerfile.latestbackup

@@ -0,0 +1,42 @@
+FROM nvidia/cuda:12.0.0-base-ubuntu20.04
+ARG DEBIAN_FRONTEND=noninteractive
+ENV PYTHONUNBUFFERED=1
+ENV PYTHON_INCLUDE /usr/include/python3.8
+ENV PYTHON_LIB /usr/lib/x86_64-linux-gnu/libpython3.8.so
+
+WORKDIR /app
+# Install python, git & ffmpeg
+RUN apt-get update && apt-get install --no-install-recommends -y \
+  build-essential \
+  python3.9 \
+  python3-pip \
+  git \
+  ffmpeg \
+  && apt-get clean && rm -rf /var/lib/apt/lists/*
+COPY ./requirements.txt /app/requirements.txt
+COPY ./packages.txt /app/packages.txt
+RUN pip install torch==1.12.1+cu113 torchvision==0.13.1+cu113 torchaudio==0.12.1+cu113 -f https://download.pytorch.org/whl/cu113/torch_stable.html
+RUN apt-get update && xargs -r -a /app/packages.txt apt-get install -y && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y python3-dev
+RUN apt-get update && apt-get install -y build-essential
+RUN which python3
+RUN which python3-config
+RUN pip3 install --no-cache-dir -r /app/requirements.txt
+RUN apt-get update && apt-get install -y build-essential
+RUN pip3 install --no-cache-dir numba==0.56.3
+RUN pip install --upgrade pip setuptools wheel
+RUN pip3 install --no-binary :all: pyworld
+RUN pip install soundfile
+
+WORKDIR /app
+# Set up a new user named "user" with user ID 1000
+RUN useradd -m -u 1000 user
+# Set the working directory to the user's home directory
+WORKDIR $HOME/app
+
+# Copy the current directory contents into the container at $HOME/app setting the owner to the user
+COPY --chown=user . $HOME/app
+
+EXPOSE 8501
+CMD nvidia-smi -l
+CMD streamlit run app.py --server.maxUploadSize 1024 --server.enableWebsocketCompression=false --server.enableXsrfProtection=false

LICENSE.md

@@ -0,0 +1,661 @@
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.

README.md

@@ -1,12 +1,10 @@
 ---
-title: Diffsvc Test
-emoji: 🔥
-colorFrom: yellow
-colorTo: red
-sdk: gradio
-sdk_version: 3.16.2
-app_file: app.py
+title: DiffSVC Inference
+emoji: 🎙
+colorFrom: red
+colorTo: orange
+sdk: docker
+app_port: 8501
 pinned: false
+duplicated_from: DIFF-SVCModel/Inference
 ---
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,106 @@
+import gradio as gr
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import json
+import os
+import tempfile
+import shutil
+import requests
+from pathlib import Path
+global ckpt_temp_file
+global audio_temp_file
+global config_temp_file
+###################################################
+from utils.hparams import hparams
+from preprocessing.data_gen_utils import get_pitch_parselmouth,get_pitch_crepe
+import numpy as np
+import matplotlib.pyplot as plt
+import IPython.display as ipd
+import utils
+import librosa
+import torchcrepe
+from infer import *
+import logging
+from infer_tools.infer_tool import *
+import io
+
+spk_dict = {
+    "雷电将军": {"model_name": './models/genshin/raiden.ckpt', "config_name": './models/genshin/config.yaml'}
+}
+
+project_name = "Unnamed"
+model_path = spk_dict['雷电将军']['model_name']
+config_path= spk_dict['雷电将军']['config_name']
+hubert_gpu = False
+svc_model = Svc(project_name, config_path, hubert_gpu, model_path)
+
+def vc_fn(sid, audio_record, audio_upload, tran, pndm_speedup=20):
+    print(sid)
+    if audio_upload is not None:
+        audio_path = audio_upload
+    elif audio_record is not None:
+        audio_path = audio_record
+    else:
+        return "你需要上传wav文件或使用网页内置的录音！", None
+
+    tran = int(tran)
+    pndm_speedup = int(pndm_speedup)
+    print('model loaded')
+    # demoaudio, sr = librosa.load(audio_path)
+    key = tran # 音高调整，支持正负（半音）
+    # 加速倍数
+    pndm_speedup = 20
+    wav_gen='queeeeee.wav'
+    
+    # Show the spinner and run the run_clip function inside the 'with' block
+    f0_tst, f0_pred, audio = run_clip(svc_model, file_path=audio_path, key=key, acc=pndm_speedup, use_crepe=True, use_pe=True, thre=0.05,
+                                            use_gt_mel=False, add_noise_step=500, project_name=project_name, out_path=wav_gen)
+
+    return "Success", (hparams['audio_sample_rate'], audio)
+
+
+app = gr.Blocks()
+with app:
+    with gr.Tabs():
+        with gr.TabItem("Basic"):
+            gr.Markdown(value="""
+            本模型为sovits_f0（含AI猫雷2.0音色），支持**60s以内**的**无伴奏**wav、mp3（单声道）格式，或使用**网页内置**的录音（二选一）
+            
+            转换效果取决于源音频语气、节奏是否与目标音色相近，以及音域是否超出目标音色音域范围
+            
+            猫雷音色低音音域效果不佳，如转换男声歌声，建议变调升 **6-10key** 
+            
+            该模型的 [github仓库链接](https://github.com/innnky/so-vits-svc)，如果想自己制作并训练模型可以访问这个 [github仓库](https://github.com/IceKyrin/sovits_guide)
+            """)
+            speaker_id = gr.Dropdown(label="音色", choices=['雷电将军'], value="雷电将军")
+            record_input = gr.Audio(source="microphone", label="录制你的声音", type="filepath", elem_id="audio_inputs")
+            upload_input = gr.Audio(source="upload", label="上传音频（长度小于60秒）", type="filepath",
+                                    elem_id="audio_inputs")
+            vc_transform = gr.Number(label="变调（整数，可以正负，半音数量，升高八度就是12）", value=0)
+            vc_speedup = gr.Number(label="加速倍数", value=20)
+            vc_submit = gr.Button("转换", variant="primary")
+            out_audio = gr.Audio(label="Output Audio")
+            gr.Markdown(value="""
+                        输出信息为音高平均偏差半音数量，体现转换音频的跑调情况（一般平均小于0.5个半音）
+                        """)
+            out_message = gr.Textbox(label="Output")
+            gr.Markdown(value="""f0曲线可以直观的显示跑调情况，蓝色为输入音高，橙色为合成音频的音高
+                        若**只看见橙色**，说明蓝色曲线被覆盖，转换效果较好
+                        """)
+            # f0_image = gr.Image(label="f0曲线")
+        vc_submit.click(vc_fn, [speaker_id, record_input, upload_input, vc_transform, vc_speedup],
+                        [out_message, out_audio])
+        with gr.TabItem("使用说明"):
+            gr.Markdown(value="""
+                        0、合集：https://github.com/IceKyrin/sovits_guide/blob/main/README.md
+                        1、仅支持sovit_f0（sovits2.0）模型
+                        2、自行下载hubert-soft-0d54a1f4.pt改名为hubert.pt放置于pth文件夹下（已经下好了）
+                            https://github.com/bshall/hubert/releases/tag/v0.1
+                        3、pth文件夹下放置sovits2.0的模型
+                        4、与模型配套的xxx.json，需有speaker项——人物列表
+                        5、放无伴奏的音频、或网页内置录音，不要放奇奇怪怪的格式
+                        6、仅供交流使用，不对用户行为负责
+                        """)
+
+    app.launch()

app.py.back2

@@ -0,0 +1,123 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import json
+import os
+import tempfile
+import shutil
+import requests
+from pathlib import Path
+temp_dir = os.path.expanduser("~/app")
+global ckpt_temp_file
+global audio_temp_file
+global config_temp_file
+###################################################
+from utils.hparams import hparams
+from preprocessing.data_gen_utils import get_pitch_parselmouth,get_pitch_crepe
+import numpy as np
+import matplotlib.pyplot as plt
+import IPython.display as ipd
+import utils
+import librosa
+import torchcrepe
+from infer import *
+import logging
+from infer_tools.infer_tool import *
+import io
+
+clip_completed = False
+def render_audio(ckpt_temp_file, config_temp_file, audio_temp_file, title):
+    logging.getLogger('numba').setLevel(logging.WARNING)
+    title = int(title)
+    project_name = "Unnamed"
+    model_path = ckpt_temp_file
+    config_path= config_temp_file
+    hubert_gpu=True
+    svc_model = Svc(project_name,config_path,hubert_gpu, model_path)
+    print('model loaded')
+    wav_fn = audio_temp_file
+    demoaudio, sr = librosa.load(wav_fn)
+    key = title # 音高调整，支持正负（半音）
+    # 加速倍数
+    pndm_speedup = 20
+    wav_gen='queeeeee.wav'#直接改后缀可以保存不同格式音频，如flac可无损压缩
+    
+    # Show the spinner and run the run_clip function inside the 'with' block
+    with st.spinner("Rendering Audio..."):
+      f0_tst, f0_pred, audio = run_clip(svc_model,file_path=wav_fn, key=key, acc=pndm_speedup, use_crepe=True, use_pe=True, thre=0.05,
+                                        use_gt_mel=False, add_noise_step=500,project_name=project_name,out_path=wav_gen)
+    clip_completed = True
+    if clip_completed:
+    # If the 'run_clip' function has completed, use the st.audio function to show an audio player for the file stored in the 'wav_gen' variable
+        st.audio(wav_gen)
+
+#######################################################
+st.set_page_config(
+    page_title="DiffSVC Render",
+    page_icon="🧊",
+    initial_sidebar_state="expanded",
+)
+############
+st.title('DIFF-SVC Render')
+
+###CKPT LOADER
+with tempfile.TemporaryDirectory(dir=os.path.expanduser("~/app")) as temp_dir:
+    ckpt = st.file_uploader("Choose your CKPT", type= 'ckpt')
+    # Check if user uploaded a CKPT file
+    if ckpt is not None:
+      #TEMP FUNCTION
+      with tempfile.NamedTemporaryFile(mode="wb", suffix='.ckpt', delete=False) as temp:
+        # Get the file contents as bytes
+        bytes_data = ckpt.getvalue()
+        # Write the bytes to the temporary file
+        temp.write(bytes_data)
+        ckpt_temp_file = temp.name
+        # Print the temporary file name
+        print(temp.name)
+
+    # Display the file path
+    if "ckpt_temp_file" in locals():
+        st.success("File saved to: {}".format(ckpt_temp_file))
+
+    # File uploader
+    config = st.file_uploader("Choose your config", type= 'yaml')
+
+    # Check if user uploaded a config file
+    if config is not None:
+      #TEMP FUNCTION
+      with tempfile.NamedTemporaryFile(mode="wb", suffix='.yaml', delete=False) as temp:
+        # Get the file contents as bytes
+        bytes_data = config.getvalue()
+        # Write the bytes to the temporary file
+        temp.write(bytes_data)
+        config_temp_file = temp.name
+        # Print the temporary file name
+        print(temp.name)
+
+    # Display the file path
+    if "config_temp_file" in locals():
+        st.success("File saved to: {}".format(config_temp_file))
+
+    audio = st.file_uploader("Choose your audio", type=["wav", "mp3"])
+
+    # Check if user uploaded an audio file
+    if audio is not None:
+  #TEMP FUNCTION
+      with tempfile.NamedTemporaryFile(mode="wb", suffix='.wav', delete=False) as temp:
+    # Get the file contents as bytes
+        bytes_data = audio.getvalue()
+    # Write the bytes to the temporary file
+        temp.write(bytes_data)
+        audio_temp_file = temp.name
+    # Print the temporary file name
+        print(temp.name)
+
+# Display the file path
+    if "audio_temp_file" in locals():
+        st.success("File saved to: {}".format(audio_temp_file))
+# Add a text input for the title with a default value of 0
+title = st.text_input("Key", value="0")
+# Add a button to start the rendering process
+if st.button("Render audio"):
+  render_audio(ckpt_temp_file, config_temp_file, audio_temp_file, title)

app.pyback

@@ -0,0 +1,123 @@
+import streamlit as st
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import json
+import os
+import tempfile
+import shutil
+import requests
+from pathlib import Path
+temp_dir = os.path.expanduser("/~app")
+global ckpt_temp_file
+global audio_temp_file
+global config_temp_file
+###################################################
+from utils.hparams import hparams
+from preprocessing.data_gen_utils import get_pitch_parselmouth,get_pitch_crepe
+import numpy as np
+import matplotlib.pyplot as plt
+import IPython.display as ipd
+import utils
+import librosa
+import torchcrepe
+from infer import *
+import logging
+from infer_tools.infer_tool import *
+import io
+
+clip_completed = False
+def render_audio(ckpt_temp_file, config_temp_file, audio_temp_file, title):
+    logging.getLogger('numba').setLevel(logging.WARNING)
+    title = int(title)
+    project_name = "Unnamed"
+    model_path = ckpt_temp_file
+    config_path= config_temp_file
+    hubert_gpu=True
+    svc_model = Svc(project_name,config_path,hubert_gpu, model_path)
+    print('model loaded')
+    wav_fn = audio_temp_file
+    demoaudio, sr = librosa.load(wav_fn)
+    key = title # 音高调整，支持正负（半音）
+    # 加速倍数
+    pndm_speedup = 20
+    wav_gen='queeeeee.wav'#直接改后缀可以保存不同格式音频，如flac可无损压缩
+    
+    # Show the spinner and run the run_clip function inside the 'with' block
+    with st.spinner("Rendering Audio..."):
+      f0_tst, f0_pred, audio = run_clip(svc_model,file_path=wav_fn, key=key, acc=pndm_speedup, use_crepe=True, use_pe=True, thre=0.05,
+                                        use_gt_mel=False, add_noise_step=500,project_name=project_name,out_path=wav_gen)
+    clip_completed = True
+    if clip_completed:
+    # If the 'run_clip' function has completed, use the st.audio function to show an audio player for the file stored in the 'wav_gen' variable
+        st.audio(wav_gen)
+
+#######################################################
+st.set_page_config(
+    page_title="DiffSVC Render",
+    page_icon="🧊",
+    initial_sidebar_state="expanded",
+)
+############
+st.title('DIFF-SVC Render')
+
+###CKPT LOADER
+with tempfile.TemporaryDirectory(dir=os.path.expanduser("/~app")) as temp_dir:
+    ckpt = st.file_uploader("Choose your CKPT", type= 'ckpt')
+    # Check if user uploaded a CKPT file
+    if ckpt is not None:
+      #TEMP FUNCTION
+      with tempfile.NamedTemporaryFile(mode="wb", suffix='.ckpt', delete=False, dir=temp_dir) as temp:
+        # Get the file contents as bytes
+        bytes_data = ckpt.getvalue()
+        # Write the bytes to the temporary file
+        temp.write(bytes_data)
+        ckpt_temp_file = temp.name
+        # Print the temporary file name
+        print(temp.name)
+
+    # Display the file path
+    if "ckpt_temp_file" in locals():
+        st.success("File saved to: {}".format(ckpt_temp_file))
+
+    # File uploader
+    config = st.file_uploader("Choose your config", type= 'yaml')
+
+    # Check if user uploaded a config file
+    if config is not None:
+      #TEMP FUNCTION
+      with tempfile.NamedTemporaryFile(mode="w", suffix='.yaml', delete=False, dir=temp_dir) as temp:
+        # Get the file contents as bytes
+        bytes_data = config.getvalue()
+        # Write the bytes to the temporary file
+        temp.write(bytes_data)
+        config_temp_file = temp.name
+        # Print the temporary file name
+        print(temp.name)
+
+    # Display the file path
+    if "config_temp_file" in locals():
+        st.success("File saved to: {}".format(config_temp_file))
+
+    audio = st.file_uploader("Choose your audio", type=["wav", "mp3"])
+
+    # Check if user uploaded an audio file
+    if audio is not None:
+  #TEMP FUNCTION
+      with tempfile.NamedTemporaryFile(mode="wb", suffix='.wav', delete=False, dir=temp_dir) as temp:
+    # Get the file contents as bytes
+        bytes_data = audio.getvalue()
+    # Write the bytes to the temporary file
+        temp.write(bytes_data)
+        audio_temp_file = temp.name
+    # Print the temporary file name
+        print(temp.name)
+
+# Display the file path
+    if "audio_temp_file" in locals():
+        st.success("File saved to: {}".format(audio_temp_file))
+# Add a text input for the title with a default value of 0
+title = st.text_input("Key", value="0")
+# Add a button to start the rendering process
+if st.button("Render audio"):
+  render_audio(ckpt_temp_file, config_temp_file, audio_temp_file, title)

batch.py

@@ -0,0 +1,43 @@
+import soundfile
+
+from infer_tools import infer_tool
+from infer_tools.infer_tool import Svc
+
+
+def run_clip(svc_model, key, acc, use_pe, use_crepe, thre, use_gt_mel, add_noise_step, project_name='', f_name=None,
+             file_path=None, out_path=None):
+    raw_audio_path = f_name
+    infer_tool.format_wav(raw_audio_path)
+    _f0_tst, _f0_pred, _audio = svc_model.infer(raw_audio_path, key=key, acc=acc, singer=True, use_pe=use_pe,
+                                                use_crepe=use_crepe,
+                                                thre=thre, use_gt_mel=use_gt_mel, add_noise_step=add_noise_step)
+    out_path = f'./singer_data/{f_name.split("/")[-1]}'
+    soundfile.write(out_path, _audio, 44100, 'PCM_16')
+
+
+if __name__ == '__main__':
+    # 工程文件夹名，训练时用的那个
+    project_name = "firefox"
+    model_path = f'./checkpoints/{project_name}/clean_model_ckpt_steps_100000.ckpt'
+    config_path = f'./checkpoints/{project_name}/config.yaml'
+
+    # 支持多个wav/ogg文件，放在raw文件夹下，带扩展名
+    file_names = infer_tool.get_end_file("./batch", "wav")
+    trans = [-6]  # 音高调整，支持正负（半音），数量与上一行对应，不足的自动按第一个移调参数补齐
+    # 加速倍数
+    accelerate = 50
+    hubert_gpu = True
+    cut_time = 30
+
+    # 下面不动
+    infer_tool.mkdir(["./batch", "./singer_data"])
+    infer_tool.fill_a_to_b(trans, file_names)
+
+    model = Svc(project_name, config_path, hubert_gpu, model_path)
+    count = 0
+    for f_name, tran in zip(file_names, trans):
+        print(f_name)
+        run_clip(model, key=tran, acc=accelerate, use_crepe=False, thre=0.05, use_pe=False, use_gt_mel=False,
+                 add_noise_step=500, f_name=f_name, project_name=project_name)
+        count += 1
+        print(f"process:{round(count * 100 / len(file_names), 2)}%")

checkpoints/0102_xiaoma_pe/config.yaml

@@ -0,0 +1,172 @@
+accumulate_grad_batches: 1
+audio_num_mel_bins: 80
+audio_sample_rate: 24000
+base_config:
+- configs/tts/lj/fs2.yaml
+binarization_args:
+  shuffle: false
+  with_align: true
+  with_f0: true
+  with_f0cwt: true
+  with_spk_embed: true
+  with_txt: true
+  with_wav: false
+binarizer_cls: data_gen.tts.base_binarizer.BaseBinarizer
+binary_data_dir: data/binary/xiaoma1022_24k_128hop
+check_val_every_n_epoch: 10
+clip_grad_norm: 1
+cwt_add_f0_loss: false
+cwt_hidden_size: 128
+cwt_layers: 2
+cwt_loss: l1
+cwt_std_scale: 0.8
+debug: false
+dec_ffn_kernel_size: 9
+dec_layers: 4
+decoder_type: fft
+dict_dir: ''
+dropout: 0.1
+ds_workers: 4
+dur_enc_hidden_stride_kernel:
+- 0,2,3
+- 0,2,3
+- 0,1,3
+dur_loss: mse
+dur_predictor_kernel: 3
+dur_predictor_layers: 2
+enc_ffn_kernel_size: 9
+enc_layers: 4
+encoder_K: 8
+encoder_type: fft
+endless_ds: true
+ffn_act: gelu
+ffn_padding: SAME
+fft_size: 512
+fmax: 12000
+fmin: 30
+gen_dir_name: ''
+hidden_size: 256
+hop_size: 128
+infer: false
+lambda_commit: 0.25
+lambda_energy: 0.1
+lambda_f0: 1.0
+lambda_ph_dur: 1.0
+lambda_sent_dur: 1.0
+lambda_uv: 1.0
+lambda_word_dur: 1.0
+load_ckpt: ''
+log_interval: 100
+loud_norm: false
+lr: 2.0
+max_epochs: 1000
+max_eval_sentences: 1
+max_eval_tokens: 60000
+max_frames: 5000
+max_input_tokens: 1550
+max_sentences: 100000
+max_tokens: 20000
+max_updates: 60000
+mel_loss: l1
+mel_vmax: 1.5
+mel_vmin: -6
+min_level_db: -120
+norm_type: gn
+num_ckpt_keep: 3
+num_heads: 2
+num_sanity_val_steps: 5
+num_spk: 1
+num_test_samples: 20
+num_valid_plots: 10
+optimizer_adam_beta1: 0.9
+optimizer_adam_beta2: 0.98
+out_wav_norm: false
+pitch_ar: false
+pitch_enc_hidden_stride_kernel:
+- 0,2,5
+- 0,2,5
+- 0,2,5
+pitch_extractor_conv_layers: 2
+pitch_loss: l1
+pitch_norm: log
+pitch_type: frame
+pre_align_args:
+  allow_no_txt: false
+  denoise: false
+  forced_align: mfa
+  txt_processor: en
+  use_sox: false
+  use_tone: true
+pre_align_cls: data_gen.tts.lj.pre_align.LJPreAlign
+predictor_dropout: 0.5
+predictor_grad: 0.1
+predictor_hidden: -1
+predictor_kernel: 5
+predictor_layers: 2
+prenet_dropout: 0.5
+prenet_hidden_size: 256
+pretrain_fs_ckpt: ''
+processed_data_dir: data/processed/ljspeech
+profile_infer: false
+raw_data_dir: data/raw/LJSpeech-1.1
+ref_norm_layer: bn
+reset_phone_dict: true
+save_best: false
+save_ckpt: true
+save_codes:
+- configs
+- modules
+- tasks
+- utils
+- usr
+save_f0: false
+save_gt: false
+seed: 1234
+sort_by_len: true
+stop_token_weight: 5.0
+task_cls: tasks.tts.pe.PitchExtractionTask
+test_ids:
+- 68
+- 70
+- 74
+- 87
+- 110
+- 172
+- 190
+- 215
+- 231
+- 294
+- 316
+- 324
+- 402
+- 422
+- 485
+- 500
+- 505
+- 508
+- 509
+- 519
+test_input_dir: ''
+test_num: 523
+test_set_name: test
+train_set_name: train
+use_denoise: false
+use_energy_embed: false
+use_gt_dur: false
+use_gt_f0: false
+use_pitch_embed: true
+use_pos_embed: true
+use_spk_embed: false
+use_spk_id: false
+use_split_spk_id: false
+use_uv: true
+use_var_enc: false
+val_check_interval: 2000
+valid_num: 348
+valid_set_name: valid
+vocoder: pwg
+vocoder_ckpt: ''
+warmup_updates: 2000
+weight_decay: 0
+win_size: 512
+work_dir: checkpoints/0102_xiaoma_pe

checkpoints/0102_xiaoma_pe/model_ckpt_steps_60000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1863f12324e43783089ab933edeeb969106b851e30d71019ebbaa9b82099d82a
+size 39141959

checkpoints/0109_hifigan_bigpopcs_hop128/config.yaml

@@ -0,0 +1,241 @@
+accumulate_grad_batches: 1
+adam_b1: 0.8
+adam_b2: 0.99
+amp: false
+audio_num_mel_bins: 80
+audio_sample_rate: 24000
+aux_context_window: 0
+#base_config:
+#- egs/egs_bases/singing/pwg.yaml
+#- egs/egs_bases/tts/vocoder/hifigan.yaml
+binarization_args:
+  reset_phone_dict: true
+  reset_word_dict: true
+  shuffle: false
+  trim_eos_bos: false
+  trim_sil: false
+  with_align: false
+  with_f0: true
+  with_f0cwt: false
+  with_linear: false
+  with_spk_embed: false
+  with_spk_id: true
+  with_txt: false
+  with_wav: true
+  with_word: false
+binarizer_cls: data_gen.tts.singing.binarize.SingingBinarizer
+binary_data_dir: data/binary/big_popcs_24k_hop128
+check_val_every_n_epoch: 10
+clip_grad_norm: 1
+clip_grad_value: 0
+datasets: []
+debug: false
+dec_ffn_kernel_size: 9
+dec_layers: 4
+dict_dir: ''
+disc_start_steps: 40000
+discriminator_grad_norm: 1
+discriminator_optimizer_params:
+  eps: 1.0e-06
+  lr: 0.0002
+  weight_decay: 0.0
+discriminator_params:
+  bias: true
+  conv_channels: 64
+  in_channels: 1
+  kernel_size: 3
+  layers: 10
+  nonlinear_activation: LeakyReLU
+  nonlinear_activation_params:
+    negative_slope: 0.2
+  out_channels: 1
+  use_weight_norm: true
+discriminator_scheduler_params:
+  gamma: 0.999
+  step_size: 600
+dropout: 0.1
+ds_workers: 1
+enc_ffn_kernel_size: 9
+enc_layers: 4
+endless_ds: true
+ffn_act: gelu
+ffn_padding: SAME
+fft_size: 512
+fmax: 12000
+fmin: 30
+frames_multiple: 1
+gen_dir_name: ''
+generator_grad_norm: 10
+generator_optimizer_params:
+  eps: 1.0e-06
+  lr: 0.0002
+  weight_decay: 0.0
+generator_params:
+  aux_channels: 80
+  dropout: 0.0
+  gate_channels: 128
+  in_channels: 1
+  kernel_size: 3
+  layers: 30
+  out_channels: 1
+  residual_channels: 64
+  skip_channels: 64
+  stacks: 3
+  upsample_net: ConvInUpsampleNetwork
+  upsample_params:
+    upsample_scales:
+    - 2
+    - 4
+    - 4
+    - 4
+  use_nsf: false
+  use_pitch_embed: true
+  use_weight_norm: true
+generator_scheduler_params:
+  gamma: 0.999
+  step_size: 600
+griffin_lim_iters: 60
+hidden_size: 256
+hop_size: 128
+infer: false
+lambda_adv: 1.0
+lambda_cdisc: 4.0
+lambda_energy: 0.0
+lambda_f0: 0.0
+lambda_mel: 5.0
+lambda_mel_adv: 1.0
+lambda_ph_dur: 0.0
+lambda_sent_dur: 0.0
+lambda_uv: 0.0
+lambda_word_dur: 0.0
+load_ckpt: ''
+loud_norm: false
+lr: 2.0
+max_epochs: 1000
+max_frames: 2400
+max_input_tokens: 1550
+max_samples: 8192
+max_sentences: 20
+max_tokens: 24000
+max_updates: 3000000
+max_valid_sentences: 1
+max_valid_tokens: 60000
+mel_loss: ssim:0.5|l1:0.5
+mel_vmax: 1.5
+mel_vmin: -6
+min_frames: 0
+min_level_db: -120
+num_ckpt_keep: 3
+num_heads: 2
+num_mels: 80
+num_sanity_val_steps: 5
+num_spk: 100
+num_test_samples: 0
+num_valid_plots: 10
+optimizer_adam_beta1: 0.9
+optimizer_adam_beta2: 0.98
+out_wav_norm: false
+pitch_extractor: parselmouth
+pitch_type: frame
+pre_align_args:
+  allow_no_txt: false
+  denoise: false
+  sox_resample: true
+  sox_to_wav: false
+  trim_sil: false
+  txt_processor: zh
+  use_tone: false
+pre_align_cls: data_gen.tts.singing.pre_align.SingingPreAlign
+predictor_grad: 0.0
+print_nan_grads: false
+processed_data_dir: ''
+profile_infer: false
+raw_data_dir: ''
+ref_level_db: 20
+rename_tmux: true
+rerun_gen: true
+resblock: '1'
+resblock_dilation_sizes:
+- - 1
+  - 3
+  - 5
+- - 1
+  - 3
+  - 5
+- - 1
+  - 3
+  - 5
+resblock_kernel_sizes:
+- 3
+- 7
+- 11
+resume_from_checkpoint: 0
+save_best: true
+save_codes: []
+save_f0: true
+save_gt: true
+scheduler: rsqrt
+seed: 1234
+sort_by_len: true
+stft_loss_params:
+  fft_sizes:
+  - 1024
+  - 2048
+  - 512
+  hop_sizes:
+  - 120
+  - 240
+  - 50
+  win_lengths:
+  - 600
+  - 1200
+  - 240
+  window: hann_window
+task_cls: tasks.vocoder.hifigan.HifiGanTask
+tb_log_interval: 100
+test_ids: []
+test_input_dir: ''
+test_num: 50
+test_prefixes: []
+test_set_name: test
+train_set_name: train
+train_sets: ''
+upsample_initial_channel: 512
+upsample_kernel_sizes:
+- 16
+- 16
+- 4
+- 4
+upsample_rates:
+- 8
+- 4
+- 2
+- 2
+use_cdisc: false
+use_cond_disc: false
+use_fm_loss: false
+use_gt_dur: true
+use_gt_f0: true
+use_mel_loss: true
+use_ms_stft: false
+use_pitch_embed: true
+use_ref_enc: true
+use_spec_disc: false
+use_spk_embed: false
+use_spk_id: false
+use_split_spk_id: false
+val_check_interval: 2000
+valid_infer_interval: 10000
+valid_monitor_key: val_loss
+valid_monitor_mode: min
+valid_set_name: valid
+vocoder: pwg
+vocoder_ckpt: ''
+vocoder_denoise_c: 0.0
+warmup_updates: 8000
+weight_decay: 0
+win_length: null
+win_size: 512
+window: hann
+word_size: 3000
+work_dir: checkpoints/0109_hifigan_bigpopcs_hop128

checkpoints/0109_hifigan_bigpopcs_hop128/model_ckpt_steps_1512000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1cb68f3ce0c46ba0a8b6d49718f1fffdf5bd7bcab769a986fd2fd129835cc1d1
+size 55827436

checkpoints/Unnamed/config.yaml

@@ -0,0 +1,445 @@
+K_step: 1000
+accumulate_grad_batches: 1
+audio_num_mel_bins: 128
+audio_sample_rate: 44100
+binarization_args:
+  shuffle: false
+  with_align: true
+  with_f0: true
+  with_hubert: true
+  with_spk_embed: false
+  with_wav: false
+binarizer_cls: preprocessing.SVCpre.SVCBinarizer
+binary_data_dir: data/binary/Unnamed
+check_val_every_n_epoch: 10
+choose_test_manually: false
+clip_grad_norm: 1
+config_path: training/config_nsf.yaml
+content_cond_steps: []
+cwt_add_f0_loss: false
+cwt_hidden_size: 128
+cwt_layers: 2
+cwt_loss: l1
+cwt_std_scale: 0.8
+datasets:
+- opencpop
+debug: false
+dec_ffn_kernel_size: 9
+dec_layers: 4
+decay_steps: 60000
+decoder_type: fft
+dict_dir: ''
+diff_decoder_type: wavenet
+diff_loss_type: l2
+dilation_cycle_length: 4
+dropout: 0.1
+ds_workers: 4
+dur_enc_hidden_stride_kernel:
+- 0,2,3
+- 0,2,3
+- 0,1,3
+dur_loss: mse
+dur_predictor_kernel: 3
+dur_predictor_layers: 5
+enc_ffn_kernel_size: 9
+enc_layers: 4
+encoder_K: 8
+encoder_type: fft
+endless_ds: false
+f0_bin: 256
+f0_max: 1100.0
+f0_min: 40.0
+ffn_act: gelu
+ffn_padding: SAME
+fft_size: 2048
+fmax: 16000
+fmin: 40
+fs2_ckpt: ''
+gaussian_start: true
+gen_dir_name: ''
+gen_tgt_spk_id: -1
+hidden_size: 256
+hop_size: 512
+hubert_gpu: true
+hubert_path: checkpoints/hubert/hubert_soft.pt
+infer: false
+keep_bins: 128
+lambda_commit: 0.25
+lambda_energy: 0.0
+lambda_f0: 1.0
+lambda_ph_dur: 0.3
+lambda_sent_dur: 1.0
+lambda_uv: 1.0
+lambda_word_dur: 1.0
+load_ckpt: ''
+log_interval: 100
+loud_norm: false
+lr: 0.0008
+max_beta: 0.02
+max_epochs: 3000
+max_eval_sentences: 1
+max_eval_tokens: 60000
+max_frames: 42000
+max_input_tokens: 60000
+max_sentences: 14
+max_tokens: 128000
+max_updates: 1000000
+mel_loss: ssim:0.5|l1:0.5
+mel_vmax: 1.5
+mel_vmin: -6.0
+min_level_db: -120
+no_fs2: true
+norm_type: gn
+num_ckpt_keep: 10
+num_heads: 2
+num_sanity_val_steps: 1
+num_spk: 1
+num_test_samples: 0
+num_valid_plots: 10
+optimizer_adam_beta1: 0.9
+optimizer_adam_beta2: 0.98
+out_wav_norm: false
+pe_ckpt: checkpoints/0102_xiaoma_pe/model_ckpt_steps_60000.ckpt
+pe_enable: false
+perform_enhance: true
+pitch_ar: false
+pitch_enc_hidden_stride_kernel:
+- 0,2,5
+- 0,2,5
+- 0,2,5
+pitch_extractor: parselmouth
+pitch_loss: l2
+pitch_norm: log
+pitch_type: frame
+pndm_speedup: 10
+pre_align_args:
+  allow_no_txt: false
+  denoise: false
+  forced_align: mfa
+  txt_processor: zh_g2pM
+  use_sox: true
+  use_tone: false
+pre_align_cls: data_gen.singing.pre_align.SingingPreAlign
+predictor_dropout: 0.5
+predictor_grad: 0.1
+predictor_hidden: -1
+predictor_kernel: 5
+predictor_layers: 5
+prenet_dropout: 0.5
+prenet_hidden_size: 256
+pretrain_fs_ckpt: ''
+processed_data_dir: xxx
+profile_infer: false
+raw_data_dir: data/raw/Unnamed
+ref_norm_layer: bn
+rel_pos: true
+reset_phone_dict: true
+residual_channels: 384
+residual_layers: 20
+save_best: true
+save_ckpt: true
+save_codes:
+- configs
+- modules
+- src
+- utils
+save_f0: true
+save_gt: false
+schedule_type: linear
+seed: 1234
+sort_by_len: true
+speaker_id: Unnamed
+spec_max:
+- 0.47615352272987366
+- 0.6125704050064087
+- 0.7518845796585083
+- 0.900716245174408
+- 0.8935521841049194
+- 0.9057011604309082
+- 0.9648348689079285
+- 0.9044283032417297
+- 0.9109272360801697
+- 0.9744535088539124
+- 0.9476388692855835
+- 0.9883336424827576
+- 1.0821290016174316
+- 1.046391248703003
+- 0.9829667806625366
+- 1.0163493156433105
+- 0.9825412631034851
+- 1.0021960735321045
+- 1.052114725112915
+- 1.128888726234436
+- 1.186057209968567
+- 1.112004280090332
+- 1.1282787322998047
+- 1.051572322845459
+- 1.1104764938354492
+- 1.176831603050232
+- 1.13348388671875
+- 1.1075258255004883
+- 1.1696264743804932
+- 1.0231049060821533
+- 0.9303848743438721
+- 1.1257890462875366
+- 1.1610286235809326
+- 1.0335885286331177
+- 1.0645352602005005
+- 1.0619306564331055
+- 1.1310148239135742
+- 1.1191954612731934
+- 1.1307402849197388
+- 1.2094721794128418
+- 1.2683185338974
+- 1.1212272644042969
+- 1.1781182289123535
+- 1.1501952409744263
+- 0.9884514808654785
+- 0.9226155281066895
+- 0.9469702839851379
+- 1.023751139640808
+- 1.1348609924316406
+- 1.087107539176941
+- 0.9899962544441223
+- 1.061837077140808
+- 1.0341650247573853
+- 0.9019684195518494
+- 0.7986546158790588
+- 0.7983465194702148
+- 0.7755436301231384
+- 0.701917290687561
+- 0.7639197707176208
+- 0.7503461837768555
+- 0.6701087951660156
+- 0.5326520800590515
+- 0.6320568323135376
+- 0.4748716950416565
+- 0.41016310453414917
+- 0.4754445552825928
+- 0.4267503023147583
+- 0.391481876373291
+- 0.3118276298046112
+- 0.3193877339363098
+- 0.3111794888973236
+- 0.3342774212360382
+- 0.1353837102651596
+- 0.16596835851669312
+- 0.1730986088514328
+- 0.2325316220521927
+- 0.17107760906219482
+- 0.10877621918916702
+- 0.2612082064151764
+- 0.11200784891843796
+- 0.14075303077697754
+- 0.07312829792499542
+- -0.011712555773556232
+- 0.1741427332162857
+- 0.19782507419586182
+- 0.03305494412779808
+- 0.004054426681250334
+- 0.1011907309293747
+- 0.1317272037267685
+- 0.014256341382861137
+- 0.019952761009335518
+- -0.1253873109817505
+- -0.14854255318641663
+- -0.14063480496406555
+- -0.1331133395433426
+- -0.28339776396751404
+- -0.38559386134147644
+- -0.2798943519592285
+- -0.19351321458816528
+- -0.23238061368465424
+- -0.2850213944911957
+- -0.20320385694503784
+- -0.24087588489055634
+- -0.15823237597942352
+- -0.13949760794639587
+- -0.19627133011817932
+- -0.1920071393251419
+- -0.19384469091892242
+- -0.22403620183467865
+- -0.18197931349277496
+- -0.28423866629600525
+- -0.26859334111213684
+- -0.3213472068309784
+- -0.3303631842136383
+- -0.3835512697696686
+- -0.3256210386753082
+- -0.3938714265823364
+- -0.4373253881931305
+- -0.4146285951137543
+- -0.4861420691013336
+- -0.4018196761608124
+- -0.46770456433296204
+- -0.4100344479084015
+- -0.5364681482315063
+- -0.5802102088928223
+- -0.5856970548629761
+- -0.47378262877464294
+- -0.36258620023727417
+spec_min:
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+spk_cond_steps: []
+stop_token_weight: 5.0
+task_cls: training.task.SVC_task.SVCTask
+test_ids: []
+test_input_dir: ''
+test_num: 0
+test_prefixes:
+- test
+test_set_name: test
+timesteps: 1000
+train_set_name: train
+use_crepe: false
+use_denoise: false
+use_energy_embed: false
+use_gt_dur: false
+use_gt_f0: false
+use_midi: false
+use_nsf: true
+use_pitch_embed: true
+use_pos_embed: true
+use_spk_embed: false
+use_spk_id: false
+use_split_spk_id: false
+use_uv: false
+use_var_enc: false
+use_vec: false
+val_check_interval: 1000
+valid_num: 0
+valid_set_name: valid
+vocoder: network.vocoders.nsf_hifigan.NsfHifiGAN
+vocoder_ckpt: checkpoints/nsf_hifigan/model
+warmup_updates: 2000
+wav2spec_eps: 1e-6
+weight_decay: 0
+win_size: 2048
+work_dir: checkpoints/Unnamed

checkpoints/Unnamed/config_nsf.yaml

@@ -0,0 +1,445 @@
+K_step: 1000
+accumulate_grad_batches: 1
+audio_num_mel_bins: 128
+audio_sample_rate: 44100
+binarization_args:
+  shuffle: false
+  with_align: true
+  with_f0: true
+  with_hubert: true
+  with_spk_embed: false
+  with_wav: false
+binarizer_cls: preprocessing.SVCpre.SVCBinarizer
+binary_data_dir: data/binary/Unnamed
+check_val_every_n_epoch: 10
+choose_test_manually: false
+clip_grad_norm: 1
+config_path: training/config_nsf.yaml
+content_cond_steps: []
+cwt_add_f0_loss: false
+cwt_hidden_size: 128
+cwt_layers: 2
+cwt_loss: l1
+cwt_std_scale: 0.8
+datasets:
+- opencpop
+debug: false
+dec_ffn_kernel_size: 9
+dec_layers: 4
+decay_steps: 20000
+decoder_type: fft
+dict_dir: ''
+diff_decoder_type: wavenet
+diff_loss_type: l2
+dilation_cycle_length: 4
+dropout: 0.1
+ds_workers: 4
+dur_enc_hidden_stride_kernel:
+- 0,2,3
+- 0,2,3
+- 0,1,3
+dur_loss: mse
+dur_predictor_kernel: 3
+dur_predictor_layers: 5
+enc_ffn_kernel_size: 9
+enc_layers: 4
+encoder_K: 8
+encoder_type: fft
+endless_ds: false
+f0_bin: 256
+f0_max: 1100.0
+f0_min: 40.0
+ffn_act: gelu
+ffn_padding: SAME
+fft_size: 2048
+fmax: 16000
+fmin: 40
+fs2_ckpt: ''
+gaussian_start: true
+gen_dir_name: ''
+gen_tgt_spk_id: -1
+hidden_size: 256
+hop_size: 512
+hubert_gpu: true
+hubert_path: checkpoints/hubert/hubert_soft.pt
+infer: false
+keep_bins: 128
+lambda_commit: 0.25
+lambda_energy: 0.0
+lambda_f0: 1.0
+lambda_ph_dur: 0.3
+lambda_sent_dur: 1.0
+lambda_uv: 1.0
+lambda_word_dur: 1.0
+load_ckpt: pretrain/nehito_ckpt_steps_1000000.ckpt
+log_interval: 100
+loud_norm: false
+lr: 5.0e-05
+max_beta: 0.02
+max_epochs: 3000
+max_eval_sentences: 1
+max_eval_tokens: 60000
+max_frames: 42000
+max_input_tokens: 60000
+max_sentences: 12
+max_tokens: 128000
+max_updates: 1000000
+mel_loss: ssim:0.5|l1:0.5
+mel_vmax: 1.5
+mel_vmin: -6.0
+min_level_db: -120
+no_fs2: true
+norm_type: gn
+num_ckpt_keep: 10
+num_heads: 2
+num_sanity_val_steps: 1
+num_spk: 1
+num_test_samples: 0
+num_valid_plots: 10
+optimizer_adam_beta1: 0.9
+optimizer_adam_beta2: 0.98
+out_wav_norm: false
+pe_ckpt: checkpoints/0102_xiaoma_pe/model_ckpt_steps_60000.ckpt
+pe_enable: false
+perform_enhance: true
+pitch_ar: false
+pitch_enc_hidden_stride_kernel:
+- 0,2,5
+- 0,2,5
+- 0,2,5
+pitch_extractor: parselmouth
+pitch_loss: l2
+pitch_norm: log
+pitch_type: frame
+pndm_speedup: 10
+pre_align_args:
+  allow_no_txt: false
+  denoise: false
+  forced_align: mfa
+  txt_processor: zh_g2pM
+  use_sox: true
+  use_tone: false
+pre_align_cls: data_gen.singing.pre_align.SingingPreAlign
+predictor_dropout: 0.5
+predictor_grad: 0.1
+predictor_hidden: -1
+predictor_kernel: 5
+predictor_layers: 5
+prenet_dropout: 0.5
+prenet_hidden_size: 256
+pretrain_fs_ckpt: ''
+processed_data_dir: xxx
+profile_infer: false
+raw_data_dir: data/raw/Unnamed
+ref_norm_layer: bn
+rel_pos: true
+reset_phone_dict: true
+residual_channels: 384
+residual_layers: 20
+save_best: false
+save_ckpt: true
+save_codes:
+- configs
+- modules
+- src
+- utils
+save_f0: true
+save_gt: false
+schedule_type: linear
+seed: 1234
+sort_by_len: true
+speaker_id: Unnamed
+spec_max:
+- -0.4884430170059204
+- 0.004534448496997356
+- 0.5684943795204163
+- 0.6527385115623474
+- 0.659079372882843
+- 0.7416915893554688
+- 0.844637930393219
+- 0.806076169013977
+- 0.7238750457763672
+- 0.9744535088539124
+- 0.9476388692855835
+- 0.9883336424827576
+- 1.0821290016174316
+- 1.046391248703003
+- 0.9829667806625366
+- 1.0163493156433105
+- 0.9825412631034851
+- 0.9834834337234497
+- 0.9811502695083618
+- 1.128888726234436
+- 1.186057209968567
+- 1.112004280090332
+- 1.1282787322998047
+- 1.051572322845459
+- 1.0510444641113281
+- 1.0110565423965454
+- 0.9236567616462708
+- 0.8036720156669617
+- 0.8383486270904541
+- 0.7735869288444519
+- 0.9303848743438721
+- 1.1257890462875366
+- 1.1610286235809326
+- 1.0335885286331177
+- 1.0645352602005005
+- 1.0619306564331055
+- 1.1310148239135742
+- 1.1191954612731934
+- 1.1307402849197388
+- 0.8837698698043823
+- 1.1153966188430786
+- 1.1045044660568237
+- 1.0479614734649658
+- 0.9491603374481201
+- 0.9858523011207581
+- 0.9226155281066895
+- 0.9469702839851379
+- 0.8791896104812622
+- 0.997624933719635
+- 0.9068642854690552
+- 0.9575618505477905
+- 0.8551340699195862
+- 0.8397778272628784
+- 0.8908605575561523
+- 0.7986546158790588
+- 0.7983465194702148
+- 0.6965265274047852
+- 0.640673041343689
+- 0.6690735220909119
+- 0.5631484985351562
+- 0.48587048053741455
+- 0.5326520800590515
+- 0.4286036193370819
+- 0.35252484679222107
+- 0.3290073573589325
+- 0.4754445552825928
+- 0.3632410168647766
+- 0.391481876373291
+- 0.20288512110710144
+- 0.18305960297584534
+- 0.1539602279663086
+- 0.03451670706272125
+- -0.16881510615348816
+- -0.02030198462307453
+- 0.10024689882993698
+- -0.023952053859829903
+- 0.05635542422533035
+- 0.10877621918916702
+- 0.006155031267553568
+- 0.07318088412284851
+- 0.14075303077697754
+- 0.057870157063007355
+- -0.0520513579249382
+- 0.1741427332162857
+- -0.11464552581310272
+- 0.03305494412779808
+- -0.06897418200969696
+- -0.12598733603954315
+- -0.09894973039627075
+- -0.2817802429199219
+- -0.0825519785284996
+- -0.3040400445461273
+- -0.4998124837875366
+- -0.36957985162734985
+- -0.5409602522850037
+- -0.49879470467567444
+- -0.713716983795166
+- -0.6545754671096802
+- -0.6425778865814209
+- -0.6178902387619019
+- -0.47356730699539185
+- -0.6165243983268738
+- -0.5841533541679382
+- -0.5759448409080505
+- -0.5498068332672119
+- -0.4661938548088074
+- -0.5811225771903992
+- -0.614664614200592
+- -0.3902229070663452
+- -0.7037366032600403
+- -0.7260795831680298
+- -0.7540019750595093
+- -0.8360528945922852
+- -0.8374698758125305
+- -0.8328713178634644
+- -0.9081047177314758
+- -0.9679695963859558
+- -0.9587443470954895
+- -1.0706337690353394
+- -0.9818469285964966
+- -0.8360191583633423
+- -0.9938981533050537
+- -1.0823708772659302
+- -1.0617167949676514
+- -1.1093820333480835
+- -1.1300138235092163
+- -1.2141350507736206
+- -1.3147293329238892
+spec_min:
+- -4.473258972167969
+- -4.244492530822754
+- -4.390527725219727
+- -4.209497928619385
+- -4.446024417877197
+- -4.3960185050964355
+- -4.164802551269531
+- -4.5063300132751465
+- -4.608232021331787
+- -4.251623630523682
+- -4.4799604415893555
+- -4.733210563659668
+- -4.411860466003418
+- -4.609100818634033
+- -4.726972579956055
+- -4.428761959075928
+- -4.487612247467041
+- -4.525552749633789
+- -4.480506896972656
+- -4.589383125305176
+- -4.608384132385254
+- -4.385376453399658
+- -4.816161632537842
+- -4.8706955909729
+- -4.848956108093262
+- -4.431278705596924
+- -4.999994277954102
+- -4.818373203277588
+- -4.527368068695068
+- -4.872085094451904
+- -4.894851207733154
+- -4.511948585510254
+- -4.534575939178467
+- -4.57792854309082
+- -4.444681644439697
+- -4.628803253173828
+- -4.74341344833374
+- -4.85427713394165
+- -4.723776817321777
+- -4.7166008949279785
+- -4.749168395996094
+- -4.67240047454834
+- -4.590690612792969
+- -4.576009750366211
+- -4.542308330535889
+- -4.890907287597656
+- -4.613001823425293
+- -4.494126796722412
+- -4.474257946014404
+- -4.574635028839111
+- -4.4817585945129395
+- -4.651009559631348
+- -4.478254795074463
+- -4.523812770843506
+- -4.546536922454834
+- -4.535660266876221
+- -4.470296859741211
+- -4.577486991882324
+- -4.541748046875
+- -4.428532123565674
+- -4.461862564086914
+- -4.489077091217041
+- -4.515830039978027
+- -4.395663738250732
+- -4.439975738525391
+- -4.4290876388549805
+- -4.397741794586182
+- -4.478252410888672
+- -4.399686336517334
+- -4.45617151260376
+- -4.434477806091309
+- -4.442898750305176
+- -4.5840277671813965
+- -4.537542819976807
+- -4.492046356201172
+- -4.534677505493164
+- -4.477104187011719
+- -4.511618614196777
+- -4.387601375579834
+- -4.499236106872559
+- -4.3717169761657715
+- -4.4242024421691895
+- -4.4055657386779785
+- -4.429355144500732
+- -4.4636993408203125
+- -4.508528232574463
+- -4.515079498291016
+- -4.426190376281738
+- -4.433525085449219
+- -4.4200215339660645
+- -4.421280860900879
+- -4.400143623352051
+- -4.419166088104248
+- -4.429825305938721
+- -4.436781406402588
+- -4.51550817489624
+- -4.518474578857422
+- -4.495880603790283
+- -4.483924865722656
+- -4.409562587738037
+- -4.3811845779418945
+- -4.411908149719238
+- -4.427165985107422
+- -4.396549701690674
+- -4.340637683868408
+- -4.405435085296631
+- -4.367630481719971
+- -4.419083595275879
+- -4.389026165008545
+- -4.371067047119141
+- -4.370710372924805
+- -4.3755269050598145
+- -4.39500093460083
+- -4.451773166656494
+- -4.365351676940918
+- -4.348028182983398
+- -4.408270359039307
+- -4.390385627746582
+- -4.347931861877441
+- -4.378237247467041
+- -4.426717758178711
+- -4.364233493804932
+- -4.371546745300293
+- -4.402477264404297
+- -4.430750846862793
+- -4.404538154602051
+- -4.384459018707275
+- -4.401677131652832
+spk_cond_steps: []
+stop_token_weight: 5.0
+task_cls: training.task.SVC_task.SVCTask
+test_ids: []
+test_input_dir: ''
+test_num: 0
+test_prefixes:
+- test
+test_set_name: test
+timesteps: 1000
+train_set_name: train
+use_crepe: false
+use_denoise: false
+use_energy_embed: false
+use_gt_dur: false
+use_gt_f0: false
+use_midi: false
+use_nsf: true
+use_pitch_embed: true
+use_pos_embed: true
+use_spk_embed: false
+use_spk_id: false
+use_split_spk_id: false
+use_uv: false
+use_var_enc: false
+use_vec: false
+val_check_interval: 1000
+valid_num: 0
+valid_set_name: valid
+vocoder: network.vocoders.nsf_hifigan.NsfHifiGAN
+vocoder_ckpt: checkpoints/nsf_hifigan/model
+warmup_updates: 2000
+wav2spec_eps: 1e-6
+weight_decay: 0
+win_size: 2048
+work_dir: checkpoints/HokoHifi

checkpoints/Unnamed/lightning_logs/lastest/hparams.yaml

@@ -0,0 +1 @@
+{}

checkpoints/Unnamed/model_ckpt_steps_192000.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c441462923580893a6170dd00126084be0a20b387b1c4fb1860755acd36c881b
+size 391390823

checkpoints/hubert/hubert_soft.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e82e7d079df05fe3aa535f6f7d42d309bdae1d2a53324e2b2386c56721f4f649
+size 378435957

checkpoints/nsf_hifigan/NOTICE.txt

@@ -0,0 +1,74 @@
+--- DiffSinger Community Vocoder ---
+
+ARCHITECTURE: NSF-HiFiGAN
+RELEASE DATE: 2022-12-11
+
+HYPER PARAMETERS:
+ - 44100 sample rate
+ - 128 mel bins
+ - 512 hop size
+ - 2048 window size
+ - fmin at 40Hz
+ - fmax at 16000Hz
+
+
+NOTICE:
+
+All model weights in the [DiffSinger Community Vocoder Project](https://openvpi.github.io/vocoders/), including
+model weights in this directory, are provided by the [OpenVPI Team](https://github.com/openvpi/), under the
+[Attribution-NonCommercial-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-nc-sa/4.0/) license.
+
+
+ACKNOWLEDGEMENTS:
+
+Training data of this vocoder is provided and permitted by the following organizations, societies and individuals:
+
+孙飒              https://www.qfssr.cn
+赤松_Akamatsu     https://www.zhibin.club
+乐威              https://www.zhibin.club
+伯添              https://space.bilibili.com/24087011
+雲宇光             https://space.bilibili.com/660675050
+橙子言             https://space.bilibili.com/318486464
+人衣大人           https://space.bilibili.com/2270344
+玖蝶              https://space.bilibili.com/676771003
+Yuuko
+白夜零BYL          https://space.bilibili.com/1605040503
+嗷天              https://space.bilibili.com/5675252
+洛泠羽            https://space.bilibili.com/347373318
+灰条纹的灰猫君      https://space.bilibili.com/2083633
+幽寂              https://space.bilibili.com/478860
+恶魔王女           https://space.bilibili.com/2475098
+AlexYHX 芮晴
+绮萱              https://y.qq.com/n/ryqq/singer/003HjD6H4aZn1K
+诗芸              https://y.qq.com/n/ryqq/singer/0005NInj142zm0
+汐蕾              https://y.qq.com/n/ryqq/singer/0023cWMH1Bq1PJ
+1262917464
+炜阳
+叶卡yolka
+幸の夏            https://space.bilibili.com/1017297686
+暮色未量           https://space.bilibili.com/272904686
+晓寞sama          https://space.bilibili.com/3463394
+没头绪的节操君
+串串BunC          https://space.bilibili.com/95817834
+落雨              https://space.bilibili.com/1292427
+长尾巴的翎艾        https://space.bilibili.com/1638666
+声闻计划           https://space.bilibili.com/392812269
+唐家大小姐         http://5sing.kugou.com/palmusic/default.html
+不伊子
+
+Training machines are provided by:
+
+花儿不哭           https://space.bilibili.com/5760446
+
+
+TERMS OF REDISTRIBUTIONS:
+
+1. Do not sell this vocoder, or charge any fees from redistributing it, as prohibited by
+   the license.
+2. Include a copy of the CC BY-NC-SA 4.0 license, or a link referring to it.
+3. Include a copy of this notice, or any other notices informing that this vocoder is
+   provided by the OpenVPI Team, that this vocoder is licensed under CC BY-NC-SA 4.0, and
+   with a complete acknowledgement list as shown above.
+4. If you fine-tuned or modified the weights, leave a notice about what has been changed.
+5. (Optional) Leave a link to the official release page of the vocoder, and tell users
+   that other versions and future updates of this vocoder can be obtained from the website.

checkpoints/nsf_hifigan/NOTICE.zh-CN.txt

@@ -0,0 +1,72 @@
+--- DiffSinger 社区声码器 ---
+
+架构：NSF-HiFiGAN
+发布日期：2022-12-11
+
+超参数：
+ - 44100 sample rate
+ - 128 mel bins
+ - 512 hop size
+ - 2048 window size
+ - fmin at 40Hz
+ - fmax at 16000Hz
+
+
+注意事项：
+
+[DiffSinger 社区声码器企划](https://openvpi.github.io/vocoders/) 中的所有模型权重，
+包括此目录下的模型权重，均由 [OpenVPI Team](https://github.com/openvpi/) 提供，并基于
+[Attribution-NonCommercial-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-nc-sa/4.0/)
+进行许可。
+
+
+致谢：
+
+此声码器的训练数据由以下组织、社团和个人提供并许可：
+
+孙飒              https://www.qfssr.cn
+赤松_Akamatsu     https://www.zhibin.club
+乐威              https://www.zhibin.club
+伯添              https://space.bilibili.com/24087011
+雲宇光             https://space.bilibili.com/660675050
+橙子言             https://space.bilibili.com/318486464
+人衣大人           https://space.bilibili.com/2270344
+玖蝶              https://space.bilibili.com/676771003
+Yuuko
+白夜零BYL          https://space.bilibili.com/1605040503
+嗷天              https://space.bilibili.com/5675252
+洛泠羽            https://space.bilibili.com/347373318
+灰条纹的灰猫君      https://space.bilibili.com/2083633
+幽寂              https://space.bilibili.com/478860
+恶魔王女           https://space.bilibili.com/2475098
+AlexYHX 芮晴
+绮萱              https://y.qq.com/n/ryqq/singer/003HjD6H4aZn1K
+诗芸              https://y.qq.com/n/ryqq/singer/0005NInj142zm0
+汐蕾              https://y.qq.com/n/ryqq/singer/0023cWMH1Bq1PJ
+1262917464
+炜阳
+叶卡yolka
+幸の夏            https://space.bilibili.com/1017297686
+暮色未量           https://space.bilibili.com/272904686
+晓寞sama          https://space.bilibili.com/3463394
+没头绪的节操君
+串串BunC          https://space.bilibili.com/95817834
+落雨              https://space.bilibili.com/1292427
+长尾巴的翎艾        https://space.bilibili.com/1638666
+声闻计划           https://space.bilibili.com/392812269
+唐家大小姐         http://5sing.kugou.com/palmusic/default.html
+不伊子
+
+训练算力的提供者如下：
+
+花儿不哭           https://space.bilibili.com/5760446
+
+
+二次分发条款：
+
+1. 请勿售卖此声码器或从其二次分发过程中收取任何费用，因为此类行为受到许可证的禁止。
+2. 请在二次分发文件中包含一份 CC BY-NC-SA 4.0 许可证的副本或指向该许可证的链接。
+3. 请在二次分发文件中包含这份声明，或以其他形式声明此声码器由 OpenVPI Team 提供并基于 CC BY-NC-SA 4.0 许可，
+   并附带上述完整的致谢名单。
+4. 如果您微调或修改了权重，请留下一份关于其受到了何种修改的说明。
+5.（可选）留下一份指向此声码器的官方发布页面的链接，并告知使用者可从该网站获取此声码器的其他版本和未来的更新。

checkpoints/nsf_hifigan/config.json

@@ -0,0 +1,38 @@
+{
+    "resblock": "1",
+    "num_gpus": 4,
+    "batch_size": 10,
+    "learning_rate": 0.0002,
+    "adam_b1": 0.8,
+    "adam_b2": 0.99,
+    "lr_decay": 0.999,
+    "seed": 1234,
+
+    "upsample_rates":        [ 8, 8, 2, 2, 2],
+    "upsample_kernel_sizes": [16,16, 4, 4, 4],
+    "upsample_initial_channel": 512,
+    "resblock_kernel_sizes": [3,7,11],
+    "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
+    "discriminator_periods": [3, 5, 7, 11, 17, 23, 37],
+
+    "segment_size": 16384,
+    "num_mels": 128,
+    "num_freq": 1025,
+    "n_fft"   : 2048,
+    "hop_size": 512,
+    "win_size": 2048,
+
+    "sampling_rate": 44100,
+
+    "fmin": 40,
+    "fmax": 16000,
+    "fmax_for_loss": null,
+
+    "num_workers": 16,
+
+    "dist_config": {
+        "dist_backend": "nccl",
+        "dist_url": "tcp://localhost:54321",
+        "world_size": 1
+    }
+}

config.yaml

@@ -0,0 +1,349 @@
+K_step: 1000
+accumulate_grad_batches: 1
+audio_num_mel_bins: 80
+audio_sample_rate: 24000
+binarization_args:
+  shuffle: false
+  with_align: true
+  with_f0: true
+  with_hubert: true
+  with_spk_embed: false
+  with_wav: false
+binarizer_cls: preprocessing.SVCpre.SVCBinarizer
+binary_data_dir: data/binary/atri
+check_val_every_n_epoch: 10
+choose_test_manually: false
+clip_grad_norm: 1
+config_path: training/config.yaml
+content_cond_steps: []
+cwt_add_f0_loss: false
+cwt_hidden_size: 128
+cwt_layers: 2
+cwt_loss: l1
+cwt_std_scale: 0.8
+datasets:
+- opencpop
+debug: false
+dec_ffn_kernel_size: 9
+dec_layers: 4
+decay_steps: 30000
+decoder_type: fft
+dict_dir: ''
+diff_decoder_type: wavenet
+diff_loss_type: l2
+dilation_cycle_length: 4
+dropout: 0.1
+ds_workers: 4
+dur_enc_hidden_stride_kernel:
+- 0,2,3
+- 0,2,3
+- 0,1,3
+dur_loss: mse
+dur_predictor_kernel: 3
+dur_predictor_layers: 5
+enc_ffn_kernel_size: 9
+enc_layers: 4
+encoder_K: 8
+encoder_type: fft
+endless_ds: False
+f0_bin: 256
+f0_max: 1100.0
+f0_min: 50.0
+ffn_act: gelu
+ffn_padding: SAME
+fft_size: 512
+fmax: 12000
+fmin: 30
+fs2_ckpt: ''
+gaussian_start: true
+gen_dir_name: ''
+gen_tgt_spk_id: -1
+hidden_size: 256
+hop_size: 128
+hubert_gpu: true
+hubert_path: checkpoints/hubert/hubert_soft.pt
+infer: false
+keep_bins: 80
+lambda_commit: 0.25
+lambda_energy: 0.0
+lambda_f0: 1.0
+lambda_ph_dur: 0.3
+lambda_sent_dur: 1.0
+lambda_uv: 1.0
+lambda_word_dur: 1.0
+load_ckpt: ''
+log_interval: 100
+loud_norm: false
+lr: 5.0e-05
+max_beta: 0.02
+max_epochs: 3000
+max_eval_sentences: 1
+max_eval_tokens: 60000
+max_frames: 42000
+max_input_tokens: 60000
+max_sentences: 24
+max_tokens: 128000
+max_updates: 1000000
+mel_loss: ssim:0.5|l1:0.5
+mel_vmax: 1.5
+mel_vmin: -6.0
+min_level_db: -120
+norm_type: gn
+num_ckpt_keep: 10
+num_heads: 2
+num_sanity_val_steps: 1
+num_spk: 1
+num_test_samples: 0
+num_valid_plots: 10
+optimizer_adam_beta1: 0.9
+optimizer_adam_beta2: 0.98
+out_wav_norm: false
+pe_ckpt: checkpoints/0102_xiaoma_pe/model_ckpt_steps_60000.ckpt
+pe_enable: false
+perform_enhance: true
+pitch_ar: false
+pitch_enc_hidden_stride_kernel:
+- 0,2,5
+- 0,2,5
+- 0,2,5
+pitch_extractor: parselmouth
+pitch_loss: l2
+pitch_norm: log
+pitch_type: frame
+pndm_speedup: 10
+pre_align_args:
+  allow_no_txt: false
+  denoise: false
+  forced_align: mfa
+  txt_processor: zh_g2pM
+  use_sox: true
+  use_tone: false
+pre_align_cls: data_gen.singing.pre_align.SingingPreAlign
+predictor_dropout: 0.5
+predictor_grad: 0.1
+predictor_hidden: -1
+predictor_kernel: 5
+predictor_layers: 5
+prenet_dropout: 0.5
+prenet_hidden_size: 256
+pretrain_fs_ckpt: pretrain/nyaru/model_ckpt_steps_60000.ckpt
+processed_data_dir: xxx
+profile_infer: false
+raw_data_dir: data/raw/atri
+ref_norm_layer: bn
+rel_pos: true
+reset_phone_dict: true
+residual_channels: 256
+residual_layers: 20
+save_best: false
+save_ckpt: true
+save_codes:
+- configs
+- modules
+- src
+- utils
+save_f0: true
+save_gt: false
+schedule_type: linear
+seed: 1234
+sort_by_len: true
+speaker_id: atri
+spec_max:
+- 0.2987259328365326
+- 0.29721200466156006
+- 0.23978209495544434
+- 0.208412766456604
+- 0.25777050852775574
+- 0.2514476478099823
+- 0.1129382848739624
+- 0.03415697440505028
+- 0.09860049188137054
+- 0.10637332499027252
+- 0.13287633657455444
+- 0.19744250178337097
+- 0.10040587931871414
+- 0.13735432922840118
+- 0.15107455849647522
+- 0.17196381092071533
+- 0.08298977464437485
+- 0.0632769986987114
+- 0.02723858878016472
+- -0.001819317927584052
+- -0.029565516859292984
+- -0.023574354127049446
+- -0.01633293740451336
+- 0.07143621146678925
+- 0.021580500528216362
+- 0.07257916033267975
+- -0.024349519982933998
+- -0.06165708228945732
+- -0.10486568510532379
+- -0.1363687664270401
+- -0.13333871960639954
+- -0.13955898582935333
+- -0.16613495349884033
+- -0.17636367678642273
+- -0.2786925733089447
+- -0.22967253625392914
+- -0.31897130608558655
+- -0.18007366359233856
+- -0.29366692900657654
+- -0.2871025800704956
+- -0.36748355627059937
+- -0.46071451902389526
+- -0.5464922189712524
+- -0.5719417333602905
+- -0.6020897626876831
+- -0.6239874958992004
+- -0.5653440952301025
+- -0.6508013606071472
+- -0.628247857093811
+- -0.6809687614440918
+- -0.569259762763977
+- -0.5423558354377747
+- -0.5811785459518433
+- -0.5359002351760864
+- -0.6565515398979187
+- -0.7143737077713013
+- -0.8502675890922546
+- -0.7979224920272827
+- -0.7110578417778015
+- -0.763409435749054
+- -0.7984790802001953
+- -0.6927220821380615
+- -0.658117413520813
+- -0.7486468553543091
+- -0.5949879884719849
+- -0.7494576573371887
+- -0.7400822639465332
+- -0.6822793483734131
+- -0.7773582339286804
+- -0.661201536655426
+- -0.791329026222229
+- -0.8982341885566711
+- -0.8736728429794312
+- -0.7701027393341064
+- -0.8490535616874695
+- -0.7479292154312134
+- -0.9320166110992432
+- -1.2862414121627808
+- -2.8936190605163574
+- -2.924229860305786
+spec_min:
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -6.0
+- -5.999454021453857
+- -5.8822431564331055
+- -5.892064571380615
+- -5.882402420043945
+- -5.786972522735596
+- -5.746835231781006
+- -5.8594512939453125
+- -5.7389445304870605
+- -5.718059539794922
+- -5.779720306396484
+- -5.801984786987305
+- -6.0
+- -6.0
+spk_cond_steps: []
+stop_token_weight: 5.0
+task_cls: training.task.SVC_task.SVCTask
+test_ids: []
+test_input_dir: ''
+test_num: 0
+test_prefixes:
+- test
+test_set_name: test
+timesteps: 1000
+train_set_name: train
+use_crepe: true
+use_denoise: false
+use_energy_embed: false
+use_gt_dur: false
+use_gt_f0: false
+use_midi: false
+use_nsf: true
+use_pitch_embed: true
+use_pos_embed: true
+use_spk_embed: false
+use_spk_id: false
+use_split_spk_id: false
+use_uv: false
+use_var_enc: false
+use_vec: false
+val_check_interval: 2000
+valid_num: 0
+valid_set_name: valid
+vocoder: network.vocoders.hifigan.HifiGAN
+vocoder_ckpt: checkpoints/0109_hifigan_bigpopcs_hop128
+warmup_updates: 2000
+wav2spec_eps: 1e-6
+weight_decay: 0
+win_size: 512
+work_dir: checkpoints/atri
+no_fs2: false

doc/train_and_inference.markdown

@@ -0,0 +1,210 @@
+# Diff-SVC(train/inference by yourself)
+## 0.环境配置
+>注意:requirements文件已更新，目前分为3个版本，可自行选择使用。\
+1. requirements.txt 是此仓库测试的原始完整环境，Torch1.12.1+cu113,可选择直接pip 或删除其中与pytorch有关的项目(torch/torchvision)后再pip，并使用自己的torch环境
+```
+pip install -r requirements.txt
+```
+>2. (推荐)requirements_short.txt 是上述环境的手动整理版，不含torch本体，也可以直接
+```
+pip install -r requirements_short.txt
+```
+>3. 根目录下有一份@三千整理的依赖列表requirements.png，是在某品牌云服务器上跑通的，不过此torch版本已不兼容目前版本代码,但是其他部分版本可以参考，十分感谢
+
+## 1.推理
+>使用根目录下的inference.ipynb进行推理或使用经过作者适配的@小狼的infer.py\
+在第一个block中修改如下参数：
+```
+config_path='checkpoints压缩包中config.yaml的位置'
+如'./checkpoints/nyaru/config.yaml'
+config和checkpoints是一一对应的，请不要使用其他config
+
+project_name='这个项目的名称'
+如'nyaru'
+
+model_path='ckpt文件的全路径'
+如'./checkpoints/nyaru/model_ckpt_steps_112000.ckpt'
+
+hubert_gpu=True
+推理时是否使用gpu推理hubert(模型中的一个模块)，不影响模型的其他部分
+目前版本已大幅减小hubert的gpu占用，在1060 6G显存下可完整推理，不需要关闭了。
+另外现已支持长音频自动切片功能(ipynb和infer.py均可)，超过30s的音频将自动在静音处切片处理，感谢@小狼的代码
+
+```
+### 可调节参数：
+```
+wav_fn='xxx.wav'#传入音频的路径，默认在项目根目录中
+
+use_crepe=True 
+#crepe是一个F0算法，效果好但速度慢，改成False会使用效果稍逊于crepe但较快的parselmouth算法
+
+thre=0.05
+#crepe的噪声过滤阈值，源音频干净可适当调大，噪音多就保持这个数值或者调小，前面改成False后这个参数不起作用
+
+pndm_speedup=20
+#推理加速算法倍数，默认是1000步，这里填成10就是只使用100步合成，是一个中规中矩的数值，这个数值可以高到50倍(20步合成)没有明显质量损失，再大可能会有可观的质量损失,注意如果下方开启了use_gt_mel, 应保证这个数值小于add_noise_step，并尽量让其能够整除
+
+key=0
+#变调参数，默认为0(不是1!!)，将源音频的音高升高key个半音后合成，如男声转女生，可填入8或者12等(12就是升高一整个8度)
+
+use_pe=True
+#梅尔谱合成音频时使用的F0提取算法，如果改成False将使用源音频的F0\
+这里填True和False合成会略有差异，通常是True会好些，但也不尽然，对合成速度几乎无影响\
+(无论key填什么 这里都是可以自由选择的，不影响)\
+44.1kHz下不支持此功能，会自动关闭，开着也不报错就是了
+
+use_gt_mel=False
+#这个选项类似于AI画图的图生图功能，如果打开，产生的音频将是输入声音与目标说话人声音的混合，混合比例由下一个参数确定
+注意!!!：这个参数如果改成True，请确保key填成0，不支持变调
+
+add_noise_step=500
+#与上个参数有关，控制两种声音的比例，填入1是完全的源声线，填入1000是完全的目标声线，能听出来是两者均等混合的数值大约在300附近(并不是线性的，另外这个参数如果调的很小，可以把pndm加速倍率调低，增加合成质量)
+
+wav_gen='yyy.wav'#输出音频的路径，默认在项目根目录中，可通过改变扩展名更改保存文件类型
+```
+如果使用infer.py，修改方式类似，需要修改__name__=='__main__'中的部分，然后在根目录中执行\
+python infer.py\
+这种方式需要将原音频放入raw中并在results中查找结果
+## 2.数据预处理与训练
+### 2.1 准备数据
+>目前支持wav格式和ogg格式的音频数据，采样率最好高于24kHz，程序会自动处理采样率和声道问题。采样率不可低于16kHz（一般不会的）\
+音频需要切片为5-15s为宜的短音频，长度没有具体要求，但不宜过长过短。音频需要为纯目标人干声，不可以有背景音乐和其他人声音，最好也不要有过重的混响等。若经过去伴奏等处理，请尽量保证处理后的音频质量。\
+目前仅支持单人训练，总时长尽量保证在3h或以上，不需要额外任何标注，将音频文件放在下述raw_data_dir下即可，这个目录下的结构可以自由定义，程序会自主找到所需文件。
+
+### 2.2 修改超参数配置
+>首先请备份一份config.yaml(此文件对应24kHz声码器, 44.1kHz声码器请使用config_nsf.yaml)，然后修改它\
+可能会用到的参数如下(以工程名为nyaru为例):
+```
+K_step: 1000
+#diffusion过程总的step,建议不要修改
+
+binary_data_dir: data/binary/nyaru
+预处理后数据的存放地址:需要将后缀改成工程名字
+
+config_path: training/config.yaml
+你要使用的这份yaml自身的地址，由于预处理过程中会写入数据，所以这个地址务必修改成将要存放这份yaml文件的完整路径
+
+choose_test_manually: false
+手动选择测试集，默认关闭，自动随机抽取5条音频作为测试集。
+如果改为ture，请在test_prefixes:中填入测试数据的文件名前缀，程序会将以对应前缀开头的文件作为测试集
+这是个列表，可以填多个前缀，如：
+test_prefixes:
+- test
+- aaaa
+- 5012
+- speaker1024
+重要：测试集*不可以*为空，为了不产生意外影响，建议尽量不要手动选择测试集
+
+endless_ds:False
+如果你的数据集过小，每个epoch时间很短，请将此项打开，将把正常的1000epoch作为一个epoch计算
+
+hubert_path: checkpoints/hubert/hubert.pt
+hubert模型的存放地址，确保这个路径是对的，一般解压checkpoints包之后就是这个路径不需要改,现已使用torch版本推理
+hubert_gpu:True
+是否在预处理时使用gpu运行hubert(模型的一个模块)，关闭后使用cpu，但耗时会显著增加。另外模型训练完推理时hubert是否用gpu是在inference中单独控制的，不受此处影响。目前hubert改为torch版后已经可以做到在1060 6G显存gpu上进行预处理，与直接推理1分钟内的音频不超出显存限制，一般不需要关了。
+
+lr: 0.0008
+#初始的学习率:这个数字对应于88的batchsize，如果batchsize更小，可以调低这个数值一些
+
+decay_steps: 20000
+每20000步学习率衰减为原来的一半，如果batchsize比较小，请调大这个数值
+
+#对于30-40左右的batchsize，推荐lr=0.0004，decay_steps=40000
+
+max_frames: 42000
+max_input_tokens: 6000
+max_sentences: 88
+max_tokens: 128000
+#batchsize是由这几个参数动态算出来的，如果不太清楚具体含义，可以只改动max_sentences这个参数，填入batchsize的最大限制值，以免炸显存
+
+pe_ckpt: checkpoints/0102_xiaoma_pe/model_ckpt_steps_60000.ckpt
+#pe模型路径，确保这个文件存在，具体作用参考inference部分
+
+raw_data_dir: data/raw/nyaru
+#存放预处理前原始数据的位置，请将原始wav数据放在这个目录下，内部文件结构无所谓，会自动解构
+
+residual_channels: 384
+residual_layers: 20
+#控制核心网络规模的一组参数，越大参数越多炼的越慢，但效果不一定会变好，大一点的数据集可以把第一个改成512。这个可以自行实验效果，不过不了解的话尽量不动。
+
+speaker_id: nyaru
+#训练的说话人名字，目前只支持单说话人，请在这里填写（只是观赏作用，没有实际意义的参数）
+
+use_crepe: true
+#在数据预处理中使用crepe提取F0,追求效果请打开，追求速度可以关闭
+
+val_check_interval: 2000
+#每2000steps推理测试集并保存ckpt
+
+vocoder_ckpt:checkpoints/0109_hifigan_bigpopcs_hop128
+#24kHz下为对应声码器的目录, 44.1kHz下为对应声码器的文件名, 注意不要填错
+
+work_dir: checkpoints/nyaru
+#修改后缀为工程名(也可以删掉或完全留空自动生成，但别乱填)
+no_fs2: true
+#对网络encoder的精简，能缩减模型体积，加快训练，且并未发现有对网络表现损害的直接证据。默认打开
+
+```
+>其他的参数如果你不知道它是做什么的，请不要修改，即使你看着名称可能以为你知道它是做什么的。
+
+### 2.3 数据预处理
+在diff-svc的目录下执行以下命令：\
+#windows
+```
+set PYTHONPATH=.
+set CUDA_VISIBLE_DEVICES=0 
+python preprocessing/binarize.py --config training/config.yaml
+```
+#linux
+```
+export PYTHONPATH=.
+CUDA_VISIBLE_DEVICES=0 python preprocessing/binarize.py --config training/config.yaml
+```
+对于预处理，@小狼准备了一份可以分段处理hubert和其他特征的代码，如果正常处理显存不足，可以先python ./network/hubert/hubert_model.py
+然后再运行正常的指令，能够识别提前处理好的hubert特征
+### 2.4 训练
+#windows
+```
+set CUDA_VISIBLE_DEVICES=0 
+python run.py --config training/config.yaml --exp_name nyaru --reset  
+```
+#linux
+```
+CUDA_VISIBLE_DEVICES=0 python run.py --config training/config.yaml --exp_name nyaru --reset 
+```
+>需要将exp_name改为你的工程名，并修改config路径，请确保和预处理使用的是同一个config文件\
+*重要* ：训练完成后，若之前不是在本地数据预处理，除了需要下载对应的ckpt文件，也需要将config文件下载下来，作为推理时使用的config，不可以使用本地之前上传上去那份。因为预处理时会向config文件中写入内容。推理时要保持使用的config和预处理使用的config是同一份。
+
+
+### 2.5 可能出现的问题：
+>2.5.1 'Upsample' object has no attribute 'recompute_scale_factor'\
+此问题发现于cuda11.3对应的torch中，若出现此问题,请通过合适的方法(如ide自动跳转等)找到你的python依赖包中的torch.nn.modules.upsampling.py文件(如conda环境中为conda目录\envs\环境目录\Lib\site-packages\torch\nn\modules\upsampling.py)，修改其153-154行
+```
+return F.interpolate(input, self.size, self.scale_factor, self.mode, self.align_corners,recompute_scale_factor=self.recompute_scale_factor)
+```
+>改为
+```
+return F.interpolate(input, self.size, self.scale_factor, self.mode, self.align_corners)
+# recompute_scale_factor=self.recompute_scale_factor)
+```
+>2.5.2 no module named 'utils'\
+请在你的运行环境(如colab笔记本)中以如下方式设置:
+```
+import os
+os.environ['PYTHONPATH']='.'
+!CUDA_VISIBLE_DEVICES=0 python preprocessing/binarize.py --config training/config.yaml
+```
+注意一定要在项目文件夹的根目录中执行
+>2.5.3 cannot load library 'libsndfile.so'\
+可能会在linux环境中遇到的错误,请执行以下指令
+```
+apt-get install libsndfile1 -y
+```
+>2.5.4 cannot load import 'consume_prefix_in_state_dict_if_present'\
+torch版本过低，请更换高版本torch
+
+>2.5.5 预处理数据过慢\
+检查是否在配置中开启了use_crepe，将其关闭可显著提升速度。\
+检查配置中hubert_gpu是否开启。
+
+如有其他问题，请加入QQ频道或discord频道询问。

flask_api.py

@@ -0,0 +1,54 @@
+import io
+import logging
+
+import librosa
+import soundfile
+from flask import Flask, request, send_file
+from flask_cors import CORS
+
+from infer_tools.infer_tool import Svc
+from utils.hparams import hparams
+
+app = Flask(__name__)
+
+CORS(app)
+
+logging.getLogger('numba').setLevel(logging.WARNING)
+
+
+@app.route("/voiceChangeModel", methods=["POST"])
+def voice_change_model():
+    request_form = request.form
+    wave_file = request.files.get("sample", None)
+    # 变调信息
+    f_pitch_change = float(request_form.get("fPitchChange", 0))
+    # DAW所需的采样率
+    daw_sample = int(float(request_form.get("sampleRate", 0)))
+    speaker_id = int(float(request_form.get("sSpeakId", 0)))
+    # http获得wav文件并转换
+    input_wav_path = io.BytesIO(wave_file.read())
+    # 模型推理
+    _f0_tst, _f0_pred, _audio = model.infer(input_wav_path, key=f_pitch_change, acc=accelerate, use_pe=False,
+                                            use_crepe=False)
+    tar_audio = librosa.resample(_audio, hparams["audio_sample_rate"], daw_sample)
+    # 返回音频
+    out_wav_path = io.BytesIO()
+    soundfile.write(out_wav_path, tar_audio, daw_sample, format="wav")
+    out_wav_path.seek(0)
+    return send_file(out_wav_path, download_name="temp.wav", as_attachment=True)
+
+
+if __name__ == '__main__':
+    # 工程文件夹名，训练时用的那个
+    project_name = "firefox"
+    model_path = f'./checkpoints/{project_name}/model_ckpt_steps_188000.ckpt'
+    config_path = f'./checkpoints/{project_name}/config.yaml'
+
+    # 加速倍数
+    accelerate = 50
+    hubert_gpu = True
+
+    model = Svc(project_name, config_path, hubert_gpu, model_path)
+
+    # 此处与vst插件对应，不建议更改
+    app.run(port=6842, host="0.0.0.0", debug=False, threaded=False)

infer.py

@@ -0,0 +1,98 @@
+import io
+import time
+from pathlib import Path
+
+import librosa
+import numpy as np
+import soundfile
+
+from infer_tools import infer_tool
+from infer_tools import slicer
+from infer_tools.infer_tool import Svc
+from utils.hparams import hparams
+
+chunks_dict = infer_tool.read_temp("./infer_tools/new_chunks_temp.json")
+
+
+def run_clip(svc_model, key, acc, use_pe, use_crepe, thre, use_gt_mel, add_noise_step, project_name='', f_name=None,
+             file_path=None, out_path=None, slice_db=-40,**kwargs):
+    print(f'code version:2022-12-04')
+    use_pe = use_pe if hparams['audio_sample_rate'] == 24000 else False
+    if file_path is None:
+        raw_audio_path = f"./raw/{f_name}"
+        clean_name = f_name[:-4]
+    else:
+        raw_audio_path = file_path
+        clean_name = str(Path(file_path).name)[:-4]
+    infer_tool.format_wav(raw_audio_path)
+    wav_path = Path(raw_audio_path).with_suffix('.wav')
+    global chunks_dict
+    audio, sr = librosa.load(wav_path, mono=True,sr=None)
+    wav_hash = infer_tool.get_md5(audio)
+    if wav_hash in chunks_dict.keys():
+        print("load chunks from temp")
+        chunks = chunks_dict[wav_hash]["chunks"]
+    else:
+        chunks = slicer.cut(wav_path, db_thresh=slice_db)
+    chunks_dict[wav_hash] = {"chunks": chunks, "time": int(time.time())}
+    infer_tool.write_temp("./infer_tools/new_chunks_temp.json", chunks_dict)
+    audio_data, audio_sr = slicer.chunks2audio(wav_path, chunks)
+
+    count = 0
+    f0_tst = []
+    f0_pred = []
+    audio = []
+    for (slice_tag, data) in audio_data:
+        print(f'#=====segment start, {round(len(data) / audio_sr, 3)}s======')
+        length = int(np.ceil(len(data) / audio_sr * hparams['audio_sample_rate']))
+        raw_path = io.BytesIO()
+        soundfile.write(raw_path, data, audio_sr, format="wav")
+        if hparams['debug']:
+            print(np.mean(data), np.var(data))
+        raw_path.seek(0)
+        if slice_tag:
+            print('jump empty segment')
+            _f0_tst, _f0_pred, _audio = (
+                np.zeros(int(np.ceil(length / hparams['hop_size']))), np.zeros(int(np.ceil(length / hparams['hop_size']))),
+                np.zeros(length))
+        else:
+            _f0_tst, _f0_pred, _audio = svc_model.infer(raw_path, key=key, acc=acc, use_pe=use_pe, use_crepe=use_crepe,
+                                                        thre=thre, use_gt_mel=use_gt_mel, add_noise_step=add_noise_step)
+        fix_audio = np.zeros(length)
+        fix_audio[:] = np.mean(_audio)
+        fix_audio[:len(_audio)] = _audio[0 if len(_audio)<len(fix_audio) else len(_audio)-len(fix_audio):]
+        f0_tst.extend(_f0_tst)
+        f0_pred.extend(_f0_pred)
+        audio.extend(list(fix_audio))
+        count += 1
+    if out_path is None:
+        out_path = f'./results/{clean_name}_{key}key_{project_name}_{hparams["residual_channels"]}_{hparams["residual_layers"]}_{int(step / 1000)}k_{accelerate}x.{kwargs["format"]}'
+    soundfile.write(out_path, audio, hparams["audio_sample_rate"], 'PCM_16',format=out_path.split('.')[-1])
+    return np.array(f0_tst), np.array(f0_pred), audio
+
+
+if __name__ == '__main__':
+    # 工程文件夹名，训练时用的那个
+    project_name = "yilanqiu"
+    model_path = f'./checkpoints/{project_name}/model_ckpt_steps_246000.ckpt'
+    config_path = f'./checkpoints/{project_name}/config.yaml'
+
+    # 支持多个wav/ogg文件，放在raw文件夹下，带扩展名
+    file_names = ["青花瓷.wav"]
+    trans = [0]  # 音高调整，支持正负（半音），数量与上一行对应，不足的自动按第一个移调参数补齐
+    # 加速倍数
+    accelerate = 20
+    hubert_gpu = True
+    format='flac'
+    step = int(model_path.split("_")[-1].split(".")[0])
+
+    # 下面不动
+    infer_tool.mkdir(["./raw", "./results"])
+    infer_tool.fill_a_to_b(trans, file_names)
+
+    model = Svc(project_name, config_path, hubert_gpu, model_path)
+    for f_name, tran in zip(file_names, trans):
+        if "." not in f_name:
+            f_name += ".wav"
+        run_clip(model, key=tran, acc=accelerate, use_crepe=True, thre=0.05, use_pe=True, use_gt_mel=False,
+                 add_noise_step=500, f_name=f_name, project_name=project_name, format=format)

infer_tools/infer_tool.py

@@ -0,0 +1,342 @@
+import hashlib
+import json
+import os
+import time
+from io import BytesIO
+from pathlib import Path
+
+import librosa
+import numpy as np
+import soundfile
+import torch
+
+import utils
+from modules.fastspeech.pe import PitchExtractor
+from network.diff.candidate_decoder import FFT
+from network.diff.diffusion import GaussianDiffusion
+from network.diff.net import DiffNet
+from network.vocoders.base_vocoder import VOCODERS, get_vocoder_cls
+from preprocessing.data_gen_utils import get_pitch_parselmouth, get_pitch_crepe, get_pitch_world
+from preprocessing.hubertinfer import Hubertencoder
+from utils.hparams import hparams, set_hparams
+from utils.pitch_utils import denorm_f0, norm_interp_f0
+
+if os.path.exists("chunks_temp.json"):
+    os.remove("chunks_temp.json")
+
+def read_temp(file_name):
+    if not os.path.exists(file_name):
+        with open(file_name, "w") as f:
+            f.write(json.dumps({"info": "temp_dict"}))
+        return {}
+    else:
+        try:
+            with open(file_name, "r") as f:
+                data = f.read()
+            data_dict = json.loads(data)
+            if os.path.getsize(file_name) > 50 * 1024 * 1024:
+                f_name = file_name.split("/")[-1]
+                print(f"clean {f_name}")
+                for wav_hash in list(data_dict.keys()):
+                    if int(time.time()) - int(data_dict[wav_hash]["time"]) > 14 * 24 * 3600:
+                        del data_dict[wav_hash]
+        except Exception as e:
+            print(e)
+            print(f"{file_name} error,auto rebuild file")
+            data_dict = {"info": "temp_dict"}
+        return data_dict
+
+
+f0_dict = read_temp("./infer_tools/f0_temp.json")
+
+
+def write_temp(file_name, data):
+    with open(file_name, "w") as f:
+        f.write(json.dumps(data))
+
+
+def timeit(func):
+    def run(*args, **kwargs):
+        t = time.time()
+        res = func(*args, **kwargs)
+        print('executing \'%s\' costed %.3fs' % (func.__name__, time.time() - t))
+        return res
+
+    return run
+
+
+def format_wav(audio_path):
+    if Path(audio_path).suffix=='.wav':
+        return
+    raw_audio, raw_sample_rate = librosa.load(audio_path, mono=True,sr=None)
+    soundfile.write(Path(audio_path).with_suffix(".wav"), raw_audio, raw_sample_rate)
+
+
+def fill_a_to_b(a, b):
+    if len(a) < len(b):
+        for _ in range(0, len(b) - len(a)):
+            a.append(a[0])
+
+
+def get_end_file(dir_path, end):
+    file_lists = []
+    for root, dirs, files in os.walk(dir_path):
+        files = [f for f in files if f[0] != '.']
+        dirs[:] = [d for d in dirs if d[0] != '.']
+        for f_file in files:
+            if f_file.endswith(end):
+                file_lists.append(os.path.join(root, f_file).replace("\\", "/"))
+    return file_lists
+
+
+def mkdir(paths: list):
+    for path in paths:
+        if not os.path.exists(path):
+            os.mkdir(path)
+
+
+def get_md5(content):
+    return hashlib.new("md5", content).hexdigest()
+
+
+class Svc:
+    def __init__(self, project_name, config_name, hubert_gpu, model_path):
+        self.project_name = project_name
+        self.DIFF_DECODERS = {
+            'wavenet': lambda hp: DiffNet(hp['audio_num_mel_bins']),
+            'fft': lambda hp: FFT(
+                hp['hidden_size'], hp['dec_layers'], hp['dec_ffn_kernel_size'], hp['num_heads']),
+        }
+
+        self.model_path = model_path
+        self.dev = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+        self._ = set_hparams(config=config_name, exp_name=self.project_name, infer=True,
+                             reset=True,
+                             hparams_str='',
+                             print_hparams=False)
+
+        self.mel_bins = hparams['audio_num_mel_bins']
+        self.model = GaussianDiffusion(
+            phone_encoder=Hubertencoder(hparams['hubert_path']),
+            out_dims=self.mel_bins, denoise_fn=self.DIFF_DECODERS[hparams['diff_decoder_type']](hparams),
+            timesteps=hparams['timesteps'],
+            K_step=hparams['K_step'],
+            loss_type=hparams['diff_loss_type'],
+            spec_min=hparams['spec_min'], spec_max=hparams['spec_max'],
+        )
+        self.load_ckpt()
+        self.model.to(self.dev)
+        hparams['hubert_gpu'] = hubert_gpu
+        self.hubert = Hubertencoder(hparams['hubert_path'])
+        self.pe = PitchExtractor().to(self.dev)
+        utils.load_ckpt(self.pe, hparams['pe_ckpt'], 'model', strict=True)
+        self.pe.eval()
+        self.vocoder = get_vocoder_cls(hparams)()
+
+    def load_ckpt(self, model_name='model', force=True, strict=True):
+        utils.load_ckpt(self.model, self.model_path, model_name, force, strict)
+
+    def infer(self, in_path, key, acc, use_pe=True, use_crepe=True, thre=0.05, singer=False, **kwargs):
+        batch = self.pre(in_path, acc, use_crepe, thre)
+        spk_embed = batch.get('spk_embed') if not hparams['use_spk_id'] else batch.get('spk_ids')
+        hubert = batch['hubert']
+        ref_mels = batch["mels"]
+        energy=batch['energy']
+        mel2ph = batch['mel2ph']
+        batch['f0'] = batch['f0'] + (key / 12)
+        batch['f0'][batch['f0']>np.log2(hparams['f0_max'])]=0
+        f0 = batch['f0']
+        uv = batch['uv']
+        @timeit
+        def diff_infer():
+            outputs = self.model(
+                hubert.to(self.dev), spk_embed=spk_embed, mel2ph=mel2ph.to(self.dev), f0=f0.to(self.dev), uv=uv.to(self.dev),energy=energy.to(self.dev),
+                ref_mels=ref_mels.to(self.dev),
+                infer=True, **kwargs)
+            return outputs
+        outputs=diff_infer()
+        batch['outputs'] = self.model.out2mel(outputs['mel_out'])
+        batch['mel2ph_pred'] = outputs['mel2ph']
+        batch['f0_gt'] = denorm_f0(batch['f0'], batch['uv'], hparams)
+        if use_pe:
+            batch['f0_pred'] = self.pe(outputs['mel_out'])['f0_denorm_pred'].detach()
+        else:
+            batch['f0_pred'] = outputs.get('f0_denorm')
+        return self.after_infer(batch, singer, in_path)
+
+    @timeit
+    def after_infer(self, prediction, singer, in_path):
+        for k, v in prediction.items():
+            if type(v) is torch.Tensor:
+                prediction[k] = v.cpu().numpy()
+
+        # remove paddings
+        mel_gt = prediction["mels"]
+        mel_gt_mask = np.abs(mel_gt).sum(-1) > 0
+
+        mel_pred = prediction["outputs"]
+        mel_pred_mask = np.abs(mel_pred).sum(-1) > 0
+        mel_pred = mel_pred[mel_pred_mask]
+        mel_pred = np.clip(mel_pred, hparams['mel_vmin'], hparams['mel_vmax'])
+
+        f0_gt = prediction.get("f0_gt")
+        f0_pred = prediction.get("f0_pred")
+        if f0_pred is not None:
+            f0_gt = f0_gt[mel_gt_mask]
+        if len(f0_pred) > len(mel_pred_mask):
+            f0_pred = f0_pred[:len(mel_pred_mask)]
+        f0_pred = f0_pred[mel_pred_mask]
+        torch.cuda.is_available() and torch.cuda.empty_cache()
+
+        if singer:
+            data_path = in_path.replace("batch", "singer_data")
+            mel_path = data_path[:-4] + "_mel.npy"
+            f0_path = data_path[:-4] + "_f0.npy"
+            np.save(mel_path, mel_pred)
+            np.save(f0_path, f0_pred)
+        wav_pred = self.vocoder.spec2wav(mel_pred, f0=f0_pred)
+        return f0_gt, f0_pred, wav_pred
+
+    def temporary_dict2processed_input(self, item_name, temp_dict, use_crepe=True, thre=0.05):
+        '''
+            process data in temporary_dicts
+        '''
+
+        binarization_args = hparams['binarization_args']
+
+        @timeit
+        def get_pitch(wav, mel):
+            # get ground truth f0 by self.get_pitch_algorithm
+            global f0_dict
+            if use_crepe:
+                md5 = get_md5(wav)
+                if f"{md5}_gt" in f0_dict.keys():
+                    print("load temp crepe f0")
+                    gt_f0 = np.array(f0_dict[f"{md5}_gt"]["f0"])
+                    coarse_f0 = np.array(f0_dict[f"{md5}_coarse"]["f0"])
+                else:
+                    torch.cuda.is_available() and torch.cuda.empty_cache()
+                    gt_f0, coarse_f0 = get_pitch_crepe(wav, mel, hparams, thre)
+                f0_dict[f"{md5}_gt"] = {"f0": gt_f0.tolist(), "time": int(time.time())}
+                f0_dict[f"{md5}_coarse"] = {"f0": coarse_f0.tolist(), "time": int(time.time())}
+                write_temp("./infer_tools/f0_temp.json", f0_dict)
+            else:
+                md5 = get_md5(wav)
+                if f"{md5}_gt_harvest" in f0_dict.keys():
+                    print("load temp harvest f0")
+                    gt_f0 = np.array(f0_dict[f"{md5}_gt_harvest"]["f0"])
+                    coarse_f0 = np.array(f0_dict[f"{md5}_coarse_harvest"]["f0"])
+                else:
+                    gt_f0, coarse_f0 = get_pitch_world(wav, mel, hparams)
+                f0_dict[f"{md5}_gt_harvest"] = {"f0": gt_f0.tolist(), "time": int(time.time())}
+                f0_dict[f"{md5}_coarse_harvest"] = {"f0": coarse_f0.tolist(), "time": int(time.time())}
+                write_temp("./infer_tools/f0_temp.json", f0_dict)
+            processed_input['f0'] = gt_f0
+            processed_input['pitch'] = coarse_f0
+
+        def get_align(mel, phone_encoded):
+            mel2ph = np.zeros([mel.shape[0]], int)
+            start_frame = 0
+            ph_durs = mel.shape[0] / phone_encoded.shape[0]
+            if hparams['debug']:
+                print(mel.shape, phone_encoded.shape, mel.shape[0] / phone_encoded.shape[0])
+            for i_ph in range(phone_encoded.shape[0]):
+                end_frame = int(i_ph * ph_durs + ph_durs + 0.5)
+                mel2ph[start_frame:end_frame + 1] = i_ph + 1
+                start_frame = end_frame + 1
+
+            processed_input['mel2ph'] = mel2ph
+
+        if hparams['vocoder'] in VOCODERS:
+            wav, mel = VOCODERS[hparams['vocoder']].wav2spec(temp_dict['wav_fn'])
+        else:
+            wav, mel = VOCODERS[hparams['vocoder'].split('.')[-1]].wav2spec(temp_dict['wav_fn'])
+        processed_input = {
+            'item_name': item_name, 'mel': mel,
+            'sec': len(wav) / hparams['audio_sample_rate'], 'len': mel.shape[0]
+        }
+        processed_input = {**temp_dict, **processed_input}  # merge two dicts
+
+        if binarization_args['with_f0']:
+            get_pitch(wav, mel)
+        if binarization_args['with_hubert']:
+            st = time.time()
+            hubert_encoded = processed_input['hubert'] = self.hubert.encode(temp_dict['wav_fn'])
+            et = time.time()
+            dev = 'cuda' if hparams['hubert_gpu'] and torch.cuda.is_available() else 'cpu'
+            print(f'hubert (on {dev}) time used {et - st}')
+
+            if binarization_args['with_align']:
+                get_align(mel, hubert_encoded)
+        return processed_input
+
+    def pre(self, wav_fn, accelerate, use_crepe=True, thre=0.05):
+        if isinstance(wav_fn, BytesIO):
+            item_name = self.project_name
+        else:
+            song_info = wav_fn.split('/')
+            item_name = song_info[-1].split('.')[-2]
+        temp_dict = {'wav_fn': wav_fn, 'spk_id': self.project_name}
+
+        temp_dict = self.temporary_dict2processed_input(item_name, temp_dict, use_crepe, thre)
+        hparams['pndm_speedup'] = accelerate
+        batch = processed_input2batch([getitem(temp_dict)])
+        return batch
+
+
+def getitem(item):
+    max_frames = hparams['max_frames']
+    spec = torch.Tensor(item['mel'])[:max_frames]
+    energy = (spec.exp() ** 2).sum(-1).sqrt()
+    mel2ph = torch.LongTensor(item['mel2ph'])[:max_frames] if 'mel2ph' in item else None
+    f0, uv = norm_interp_f0(item["f0"][:max_frames], hparams)
+    hubert = torch.Tensor(item['hubert'][:hparams['max_input_tokens']])
+    pitch = torch.LongTensor(item.get("pitch"))[:max_frames]
+    sample = {
+        "item_name": item['item_name'],
+        "hubert": hubert,
+        "mel": spec,
+        "pitch": pitch,
+        "energy": energy,
+        "f0": f0,
+        "uv": uv,
+        "mel2ph": mel2ph,
+        "mel_nonpadding": spec.abs().sum(-1) > 0,
+    }
+    return sample
+
+
+def processed_input2batch(samples):
+    '''
+        Args:
+            samples: one batch of processed_input
+        NOTE:
+            the batch size is controlled by hparams['max_sentences']
+    '''
+    if len(samples) == 0:
+        return {}
+    item_names = [s['item_name'] for s in samples]
+    hubert = utils.collate_2d([s['hubert'] for s in samples], 0.0)
+    f0 = utils.collate_1d([s['f0'] for s in samples], 0.0)
+    pitch = utils.collate_1d([s['pitch'] for s in samples])
+    uv = utils.collate_1d([s['uv'] for s in samples])
+    energy = utils.collate_1d([s['energy'] for s in samples], 0.0)
+    mel2ph = utils.collate_1d([s['mel2ph'] for s in samples], 0.0) \
+        if samples[0]['mel2ph'] is not None else None
+    mels = utils.collate_2d([s['mel'] for s in samples], 0.0)
+    mel_lengths = torch.LongTensor([s['mel'].shape[0] for s in samples])
+
+    batch = {
+        'item_name': item_names,
+        'nsamples': len(samples),
+        'hubert': hubert,
+        'mels': mels,
+        'mel_lengths': mel_lengths,
+        'mel2ph': mel2ph,
+        'energy': energy,
+        'pitch': pitch,
+        'f0': f0,
+        'uv': uv,
+    }
+    return batch

infer_tools/new_chunks_temp.json

@@ -0,0 +1 @@
+{"info": "temp_dict", "accd68639783a1819e41702c4c1bf2e7": {"chunks": {"0": {"slice": false, "split_time": "0,607727"}}, "time": 1672781849}, "28b718f4ef116ca8c4d2279dfc0bd161": {"chunks": {"0": {"slice": false, "split_time": "0,607727"}}, "time": 1672758446}, "3c68f6ef87cdbea1be9b66e78bcd1c62": {"chunks": {"0": {"slice": true, "split_time": "0,20115"}, "1": {"slice": false, "split_time": "20115,152363"}, "2": {"slice": true, "split_time": "152363,163441"}, "3": {"slice": false, "split_time": "163441,347184"}, "4": {"slice": true, "split_time": "347184,351976"}, "5": {"slice": false, "split_time": "351976,438356"}, "6": {"slice": true, "split_time": "438356,499095"}}, "time": 1673478071}, "dd17f428601bccf6dd3c82c6d6daaaba": {"chunks": {"0": {"slice": true, "split_time": "0,24155"}, "1": {"slice": false, "split_time": "24155,323983"}, "2": {"slice": true, "split_time": "323983,352800"}}, "time": 1672758814}, "7d915edda42b3c65471bb0f86ba2a57c": {"chunks": {"0": {"slice": false, "split_time": "0,1417197"}, "1": {"slice": true, "split_time": "1417197,1426665"}, "2": {"slice": false, "split_time": "1426665,1736746"}, "3": {"slice": true, "split_time": "1736746,1743374"}, "4": {"slice": false, "split_time": "1743374,2042438"}, "5": {"slice": true, "split_time": "2042438,2050710"}, "6": {"slice": false, "split_time": "2050710,2508864"}, "7": {"slice": true, "split_time": "2508864,2515696"}, "8": {"slice": false, "split_time": "2515696,2682383"}}, "time": 1672772091}, "3bad51a34c1a940a31de387147e10c0a": {"chunks": {"0": {"slice": false, "split_time": "0,318420"}, "1": {"slice": true, "split_time": "318420,325186"}, "2": {"slice": false, "split_time": "325186,1581611"}, "3": {"slice": true, "split_time": "1581611,1594730"}}, "time": 1672772297}, "3ffa2635bed65fb96d8476803560fd1a": {"chunks": {"0": {"slice": true, "split_time": "0,668952"}, "1": {"slice": false, "split_time": "668952,894598"}, "2": {"slice": true, "split_time": "894598,917516"}, "3": {"slice": false, "split_time": "917516,1140044"}, "4": {"slice": true, "split_time": "1140044,1159951"}, "5": {"slice": false, "split_time": "1159951,1725379"}, "6": {"slice": true, "split_time": "1725379,1756813"}, "7": {"slice": false, "split_time": "1756813,3302878"}, "8": {"slice": true, "split_time": "3302878,3361575"}, "9": {"slice": false, "split_time": "3361575,3582626"}, "10": {"slice": true, "split_time": "3582626,3609059"}, "11": {"slice": false, "split_time": "3609059,3844622"}, "12": {"slice": true, "split_time": "3844622,3861910"}, "13": {"slice": false, "split_time": "3861910,4440673"}, "14": {"slice": true, "split_time": "4440673,4468405"}, "15": {"slice": false, "split_time": "4468405,5108832"}, "16": {"slice": true, "split_time": "5108832,5129497"}, "17": {"slice": false, "split_time": "5129497,6675968"}, "18": {"slice": true, "split_time": "6675968,9217857"}}, "time": 1673261015}, "5fba9ddc6c7223c2907d9ff3169e23f1": {"chunks": {"0": {"slice": true, "split_time": "0,53579"}, "1": {"slice": false, "split_time": "53579,332754"}, "2": {"slice": true, "split_time": "332754,414277"}, "3": {"slice": false, "split_time": "414277,781044"}, "4": {"slice": true, "split_time": "781044,816837"}, "5": {"slice": false, "split_time": "816837,1549835"}, "6": {"slice": true, "split_time": "1549835,1557319"}, "7": {"slice": false, "split_time": "1557319,4476357"}, "8": {"slice": true, "split_time": "4476357,4503731"}, "9": {"slice": false, "split_time": "4503731,5209666"}, "10": {"slice": true, "split_time": "5209666,5213405"}, "11": {"slice": false, "split_time": "5213405,6021387"}, "12": {"slice": true, "split_time": "6021387,6063481"}, "13": {"slice": false, "split_time": "6063481,6491872"}, "14": {"slice": true, "split_time": "6491872,6602234"}, "15": {"slice": false, "split_time": "6602234,7311975"}, "16": {"slice": true, "split_time": "7311975,7328295"}, "17": {"slice": false, "split_time": "7328295,8137067"}, "18": {"slice": true, "split_time": "8137067,8142127"}, "19": {"slice": false, "split_time": "8142127,10821823"}, "20": {"slice": true, "split_time": "10821823,11066085"}, "21": {"slice": false, "split_time": "11066085,11148757"}, "22": {"slice": true, "split_time": "11148757,11459584"}}, "time": 1673446113}, "a8b37529b910527ef9b414d4fa485973": {"chunks": {"0": {"slice": true, "split_time": "0,74804"}, "1": {"slice": false, "split_time": "74804,545959"}, "2": {"slice": true, "split_time": "545959,550194"}, "3": {"slice": false, "split_time": "550194,1040392"}, "4": {"slice": true, "split_time": "1040392,1045223"}, "5": {"slice": false, "split_time": "1045223,2552370"}, "6": {"slice": true, "split_time": "2552370,2825551"}, "7": {"slice": false, "split_time": "2825551,3280352"}, "8": {"slice": true, "split_time": "3280352,3284153"}, "9": {"slice": false, "split_time": "3284153,4005079"}, "10": {"slice": true, "split_time": "4005079,4049271"}, "11": {"slice": false, "split_time": "4049271,4279046"}, "12": {"slice": true, "split_time": "4279046,4323261"}, "13": {"slice": false, "split_time": "4323261,5505905"}, "14": {"slice": true, "split_time": "5505905,5535670"}, "15": {"slice": false, "split_time": "5535670,5770626"}, "16": {"slice": true, "split_time": "5770626,5829727"}, "17": {"slice": false, "split_time": "5829727,7771861"}, "18": {"slice": true, "split_time": "7771861,8040098"}, "19": {"slice": false, "split_time": "8040098,8536352"}, "20": {"slice": true, "split_time": "8536352,9186304"}}, "time": 1673040810}, "7e44079101dd10bc262ee28c4b881f25": {"chunks": {"0": {"slice": true, "split_time": "0,135941"}, "1": {"slice": false, "split_time": "135941,787855"}, "2": {"slice": true, "split_time": "787855,999653"}, "3": {"slice": false, "split_time": "999653,2981684"}, "4": {"slice": true, "split_time": "2981684,3116588"}, "5": {"slice": false, "split_time": "3116588,3339320"}, "6": {"slice": true, "split_time": "3339320,3382142"}, "7": {"slice": false, "split_time": "3382142,5345853"}, "8": {"slice": true, "split_time": "5345853,5365617"}, "9": {"slice": false, "split_time": "5365617,6391431"}, "10": {"slice": true, "split_time": "6391431,6527569"}, "11": {"slice": false, "split_time": "6527569,7791684"}, "12": {"slice": true, "split_time": "7791684,7794073"}, "13": {"slice": false, "split_time": "7794073,8749857"}, "14": {"slice": true, "split_time": "8749857,9232506"}, "15": {"slice": false, "split_time": "9232506,11714376"}, "16": {"slice": true, "split_time": "11714376,11785553"}, "17": {"slice": false, "split_time": "11785553,12282894"}, "18": {"slice": true, "split_time": "12282894,12861440"}}, "time": 1673260417}, "89ccd0cc779c17deb9dfd54e700a0bcd": {"chunks": {"0": {"slice": true, "split_time": "0,15387"}, "1": {"slice": false, "split_time": "15387,348933"}, "2": {"slice": true, "split_time": "348933,352886"}, "3": {"slice": false, "split_time": "352886,675429"}, "4": {"slice": true, "split_time": "675429,678891"}, "5": {"slice": false, "split_time": "678891,1048502"}, "6": {"slice": true, "split_time": "1048502,1056469"}, "7": {"slice": false, "split_time": "1056469,1337989"}, "8": {"slice": true, "split_time": "1337989,1381376"}}, "time": 1673351834}, "46fa6dbb81bfe1252ebdefa0884d0e6d": {"chunks": {"0": {"slice": true, "split_time": "0,185799"}, "1": {"slice": false, "split_time": "185799,502227"}, "2": {"slice": true, "split_time": "502227,848473"}, "3": {"slice": false, "split_time": "848473,2347749"}, "4": {"slice": true, "split_time": "2347749,2348783"}, "5": {"slice": false, "split_time": "2348783,3429977"}, "6": {"slice": true, "split_time": "3429977,3890042"}, "7": {"slice": false, "split_time": "3890042,5375055"}, "8": {"slice": true, "split_time": "5375055,5402621"}, "9": {"slice": false, "split_time": "5402621,6462260"}, "10": {"slice": true, "split_time": "6462260,6923811"}, "11": {"slice": false, "split_time": "6923811,8507888"}, "12": {"slice": true, "split_time": "8507888,9252348"}, "13": {"slice": false, "split_time": "9252348,10599478"}, "14": {"slice": true, "split_time": "10599478,10628316"}, "15": {"slice": false, "split_time": "10628316,10692608"}}, "time": 1673446761}, "2b0ffc08fb6fb0f29df83ae8cc0c0eb8": {"chunks": {"0": {"slice": true, "split_time": "0,1208342"}, "1": {"slice": false, "split_time": "1208342,1917473"}, "2": {"slice": true, "split_time": "1917473,1984255"}, "3": {"slice": false, "split_time": "1984255,4413108"}, "4": {"slice": true, "split_time": "4413108,4882745"}, "5": {"slice": false, "split_time": "4882745,8072787"}, "6": {"slice": true, "split_time": "8072787,9419776"}}, "time": 1673388059}, "57440513a8cfffbc7c1ffb790b9723f2": {"chunks": {"0": {"slice": true, "split_time": "0,1199064"}, "1": {"slice": false, "split_time": "1199064,1431015"}, "2": {"slice": true, "split_time": "1431015,1452361"}, "3": {"slice": false, "split_time": "1452361,1898308"}, "4": {"slice": true, "split_time": "1898308,1904596"}, "5": {"slice": false, "split_time": "1904596,2468360"}, "6": {"slice": true, "split_time": "2468360,2479240"}, "7": {"slice": false, "split_time": "2479240,3593434"}, "8": {"slice": true, "split_time": "3593434,3629186"}, "9": {"slice": false, "split_time": "3629186,4074575"}, "10": {"slice": true, "split_time": "4074575,4078966"}, "11": {"slice": false, "split_time": "4078966,4307991"}, "12": {"slice": true, "split_time": "4307991,4316241"}, "13": {"slice": false, "split_time": "4316241,5986540"}, "14": {"slice": true, "split_time": "5986540,5990713"}, "15": {"slice": false, "split_time": "5990713,6243455"}, "16": {"slice": true, "split_time": "6243455,6491661"}, "17": {"slice": false, "split_time": "6491661,6719744"}, "18": {"slice": true, "split_time": "6719744,6739748"}, "19": {"slice": false, "split_time": "6739748,7021570"}, "20": {"slice": true, "split_time": "7021570,7069439"}, "21": {"slice": false, "split_time": "7069439,7363743"}, "22": {"slice": true, "split_time": "7363743,7376200"}, "23": {"slice": false, "split_time": "7376200,9402054"}, "24": {"slice": true, "split_time": "9402054,9403936"}, "25": {"slice": false, "split_time": "9403936,9697617"}, "26": {"slice": true, "split_time": "9697617,9878400"}}, "time": 1673468608}, "1d7d07c16652e4b7300bdaf5d38128b9": {"chunks": {"0": {"slice": true, "split_time": "0,261052"}, "1": {"slice": false, "split_time": "261052,993078"}, "2": {"slice": true, "split_time": "993078,1091294"}, "3": {"slice": false, "split_time": "1091294,2303042"}, "4": {"slice": true, "split_time": "2303042,2304533"}, "5": {"slice": false, "split_time": "2304533,3543442"}, "6": {"slice": true, "split_time": "3543442,3656782"}, "7": {"slice": false, "split_time": "3656782,4337660"}, "8": {"slice": true, "split_time": "4337660,4406921"}, "9": {"slice": false, "split_time": "4406921,6132779"}, "10": {"slice": true, "split_time": "6132779,6147466"}, "11": {"slice": false, "split_time": "6147466,6855506"}, "12": {"slice": true, "split_time": "6855506,7413094"}}, "time": 1673466029}, "dc6af7b6086c4a6f7f6ee206024988f3": {"chunks": {"0": {"slice": true, "split_time": "0,1492137"}, "1": {"slice": false, "split_time": "1492137,2132538"}, "2": {"slice": true, "split_time": "2132538,2140949"}, "3": {"slice": false, "split_time": "2140949,2362096"}, "4": {"slice": true, "split_time": "2362096,2377313"}, "5": {"slice": false, "split_time": "2377313,3190602"}, "6": {"slice": true, "split_time": "3190602,3236563"}, "7": {"slice": false, "split_time": "3236563,3795750"}, "8": {"slice": true, "split_time": "3795750,3798646"}, "9": {"slice": false, "split_time": "3798646,4037700"}, "10": {"slice": true, "split_time": "4037700,4057671"}, "11": {"slice": false, "split_time": "4057671,4828341"}, "12": {"slice": true, "split_time": "4828341,4839270"}, "13": {"slice": false, "split_time": "4839270,5069319"}, "14": {"slice": true, "split_time": "5069319,5070998"}, "15": {"slice": false, "split_time": "5070998,5722290"}, "16": {"slice": true, "split_time": "5722290,5778275"}, "17": {"slice": false, "split_time": "5778275,6346775"}, "18": {"slice": true, "split_time": "6346775,6354400"}, "19": {"slice": false, "split_time": "6354400,6965771"}, "20": {"slice": true, "split_time": "6965771,7361821"}, "21": {"slice": false, "split_time": "7361821,7649206"}, "22": {"slice": true, "split_time": "7649206,7681433"}, "23": {"slice": false, "split_time": "7681433,7921525"}, "24": {"slice": true, "split_time": "7921525,7934007"}, "25": {"slice": false, "split_time": "7934007,8183308"}, "26": {"slice": true, "split_time": "8183308,8238306"}, "27": {"slice": false, "split_time": "8238306,10906572"}, "28": {"slice": true, "split_time": "10906572,10942779"}}, "time": 1673469959}, "010900becfb5574ab3a333f941577c2c": {"chunks": {"0": {"slice": true, "split_time": "0,1083101"}, "1": {"slice": false, "split_time": "1083101,1421801"}, "2": {"slice": true, "split_time": "1421801,1453133"}, "3": {"slice": false, "split_time": "1453133,1794206"}, "4": {"slice": true, "split_time": "1794206,1800891"}, "5": {"slice": false, "split_time": "1800891,2115485"}, "6": {"slice": true, "split_time": "2115485,2152253"}, "7": {"slice": false, "split_time": "2152253,2957806"}, "8": {"slice": true, "split_time": "2957806,2967552"}, "9": {"slice": false, "split_time": "2967552,3967440"}, "10": {"slice": true, "split_time": "3967440,4012384"}, "11": {"slice": false, "split_time": "4012384,4335874"}, "12": {"slice": true, "split_time": "4335874,4363031"}, "13": {"slice": false, "split_time": "4363031,4694987"}, "14": {"slice": true, "split_time": "4694987,4715486"}, "15": {"slice": false, "split_time": "4715486,5030719"}, "16": {"slice": true, "split_time": "5030719,5065596"}, "17": {"slice": false, "split_time": "5065596,6874746"}, "18": {"slice": true, "split_time": "6874746,7885915"}, "19": {"slice": false, "split_time": "7885915,9628060"}, "20": {"slice": true, "split_time": "9628060,10032061"}}, "time": 1673470410}, "5bc32d122ced9dd6d8cf0186fff6466a": {"chunks": {"0": {"slice": true, "split_time": "0,814909"}, "1": {"slice": false, "split_time": "814909,3487605"}, "2": {"slice": true, "split_time": "3487605,3496613"}, "3": {"slice": false, "split_time": "3496613,6186104"}, "4": {"slice": true, "split_time": "6186104,6826505"}, "5": {"slice": false, "split_time": "6826505,9979380"}, "6": {"slice": true, "split_time": "9979380,13711090"}}, "time": 1673474828}}

infer_tools/slicer.py

@@ -0,0 +1,158 @@
+import time
+
+import numpy as np
+import torch
+import torchaudio
+from scipy.ndimage import maximum_filter1d, uniform_filter1d
+
+
+def timeit(func):
+    def run(*args, **kwargs):
+        t = time.time()
+        res = func(*args, **kwargs)
+        print('executing \'%s\' costed %.3fs' % (func.__name__, time.time() - t))
+        return res
+
+    return run
+
+
+# @timeit
+def _window_maximum(arr, win_sz):
+    return maximum_filter1d(arr, size=win_sz)[win_sz // 2: win_sz // 2 + arr.shape[0] - win_sz + 1]
+
+
+# @timeit
+def _window_rms(arr, win_sz):
+    filtered = np.sqrt(uniform_filter1d(np.power(arr, 2), win_sz) - np.power(uniform_filter1d(arr, win_sz), 2))
+    return filtered[win_sz // 2: win_sz // 2 + arr.shape[0] - win_sz + 1]
+
+
+def level2db(levels, eps=1e-12):
+    return 20 * np.log10(np.clip(levels, a_min=eps, a_max=1))
+
+
+def _apply_slice(audio, begin, end):
+    if len(audio.shape) > 1:
+        return audio[:, begin: end]
+    else:
+        return audio[begin: end]
+
+
+class Slicer:
+    def __init__(self,
+                 sr: int,
+                 db_threshold: float = -40,
+                 min_length: int = 5000,
+                 win_l: int = 300,
+                 win_s: int = 20,
+                 max_silence_kept: int = 500):
+        self.db_threshold = db_threshold
+        self.min_samples = round(sr * min_length / 1000)
+        self.win_ln = round(sr * win_l / 1000)
+        self.win_sn = round(sr * win_s / 1000)
+        self.max_silence = round(sr * max_silence_kept / 1000)
+        if not self.min_samples >= self.win_ln >= self.win_sn:
+            raise ValueError('The following condition must be satisfied: min_length >= win_l >= win_s')
+        if not self.max_silence >= self.win_sn:
+            raise ValueError('The following condition must be satisfied: max_silence_kept >= win_s')
+
+    @timeit
+    def slice(self, audio):
+        samples = audio
+        if samples.shape[0] <= self.min_samples:
+            return {"0": {"slice": False, "split_time": f"0,{len(audio)}"}}
+        # get absolute amplitudes
+        abs_amp = np.abs(samples - np.mean(samples))
+        # calculate local maximum with large window
+        win_max_db = level2db(_window_maximum(abs_amp, win_sz=self.win_ln))
+        sil_tags = []
+        left = right = 0
+        while right < win_max_db.shape[0]:
+            if win_max_db[right] < self.db_threshold:
+                right += 1
+            elif left == right:
+                left += 1
+                right += 1
+            else:
+                if left == 0:
+                    split_loc_l = left
+                else:
+                    sil_left_n = min(self.max_silence, (right + self.win_ln - left) // 2)
+                    rms_db_left = level2db(_window_rms(samples[left: left + sil_left_n], win_sz=self.win_sn))
+                    split_win_l = left + np.argmin(rms_db_left)
+                    split_loc_l = split_win_l + np.argmin(abs_amp[split_win_l: split_win_l + self.win_sn])
+                if len(sil_tags) != 0 and split_loc_l - sil_tags[-1][1] < self.min_samples and right < win_max_db.shape[
+                    0] - 1:
+                    right += 1
+                    left = right
+                    continue
+                if right == win_max_db.shape[0] - 1:
+                    split_loc_r = right + self.win_ln
+                else:
+                    sil_right_n = min(self.max_silence, (right + self.win_ln - left) // 2)
+                    rms_db_right = level2db(_window_rms(samples[right + self.win_ln - sil_right_n: right + self.win_ln],
+                                                        win_sz=self.win_sn))
+                    split_win_r = right + self.win_ln - sil_right_n + np.argmin(rms_db_right)
+                    split_loc_r = split_win_r + np.argmin(abs_amp[split_win_r: split_win_r + self.win_sn])
+                sil_tags.append((split_loc_l, split_loc_r))
+                right += 1
+                left = right
+        if left != right:
+            sil_left_n = min(self.max_silence, (right + self.win_ln - left) // 2)
+            rms_db_left = level2db(_window_rms(samples[left: left + sil_left_n], win_sz=self.win_sn))
+            split_win_l = left + np.argmin(rms_db_left)
+            split_loc_l = split_win_l + np.argmin(abs_amp[split_win_l: split_win_l + self.win_sn])
+            sil_tags.append((split_loc_l, samples.shape[0]))
+        if len(sil_tags) == 0:
+            return {"0": {"slice": False, "split_time": f"0,{len(audio)}"}}
+        else:
+            chunks = []
+            # 第一段静音并非从头开始，补上有声片段
+            if sil_tags[0][0]:
+                chunks.append({"slice": False, "split_time": f"0,{sil_tags[0][0]}"})
+            for i in range(0, len(sil_tags)):
+                # 标识有声片段（跳过第一段）
+                if i:
+                    chunks.append({"slice": False, "split_time": f"{sil_tags[i - 1][1]},{sil_tags[i][0]}"})
+                # 标识所有静音片段
+                chunks.append({"slice": True, "split_time": f"{sil_tags[i][0]},{sil_tags[i][1]}"})
+            # 最后一段静音并非结尾，补上结尾片段
+            if sil_tags[-1][1] != len(audio):
+                chunks.append({"slice": False, "split_time": f"{sil_tags[-1][1]},{len(audio)}"})
+            chunk_dict = {}
+            for i in range(len(chunks)):
+                chunk_dict[str(i)] = chunks[i]
+            return chunk_dict
+
+
+def cut(audio_path, db_thresh=-30, min_len=5000, win_l=300, win_s=20, max_sil_kept=500):
+    audio, sr = torchaudio.load(audio_path)
+    if len(audio.shape) == 2 and audio.shape[1] >= 2:
+        audio = torch.mean(audio, dim=0).unsqueeze(0)
+    audio = audio.cpu().numpy()[0]
+
+    slicer = Slicer(
+        sr=sr,
+        db_threshold=db_thresh,
+        min_length=min_len,
+        win_l=win_l,
+        win_s=win_s,
+        max_silence_kept=max_sil_kept
+    )
+    chunks = slicer.slice(audio)
+    return chunks
+
+
+def chunks2audio(audio_path, chunks):
+    chunks = dict(chunks)
+    audio, sr = torchaudio.load(audio_path)
+    if len(audio.shape) == 2 and audio.shape[1] >= 2:
+        audio = torch.mean(audio, dim=0).unsqueeze(0)
+    audio = audio.cpu().numpy()[0]
+    result = []
+    for k, v in chunks.items():
+        tag = v["split_time"].split(",")
+        result.append((v["slice"], audio[int(tag[0]):int(tag[1])]))
+    return result, sr
+
+

models/genshin/config.yaml

@@ -0,0 +1,445 @@
+K_step: 1000
+accumulate_grad_batches: 1
+audio_num_mel_bins: 128
+audio_sample_rate: 44100
+binarization_args:
+  shuffle: false
+  with_align: true
+  with_f0: true
+  with_hubert: true
+  with_spk_embed: false
+  with_wav: false
+binarizer_cls: preprocessing.SVCpre.SVCBinarizer
+binary_data_dir: data/binary/raiden
+check_val_every_n_epoch: 10
+choose_test_manually: false
+clip_grad_norm: 1
+config_path: training/config_nsf.yaml
+content_cond_steps: []
+cwt_add_f0_loss: false
+cwt_hidden_size: 128
+cwt_layers: 2
+cwt_loss: l1
+cwt_std_scale: 0.8
+datasets:
+- opencpop
+debug: false
+dec_ffn_kernel_size: 9
+dec_layers: 4
+decay_steps: 50000
+decoder_type: fft
+dict_dir: ''
+diff_decoder_type: wavenet
+diff_loss_type: l2
+dilation_cycle_length: 4
+dropout: 0.1
+ds_workers: 4
+dur_enc_hidden_stride_kernel:
+- 0,2,3
+- 0,2,3
+- 0,1,3
+dur_loss: mse
+dur_predictor_kernel: 3
+dur_predictor_layers: 5
+enc_ffn_kernel_size: 9
+enc_layers: 4
+encoder_K: 8
+encoder_type: fft
+endless_ds: false
+f0_bin: 256
+f0_max: 1100.0
+f0_min: 40.0
+ffn_act: gelu
+ffn_padding: SAME
+fft_size: 2048
+fmax: 16000
+fmin: 40
+fs2_ckpt: ''
+gaussian_start: true
+gen_dir_name: ''
+gen_tgt_spk_id: -1
+hidden_size: 256
+hop_size: 512
+hubert_gpu: true
+hubert_path: checkpoints/hubert/hubert_soft.pt
+infer: false
+keep_bins: 128
+lambda_commit: 0.25
+lambda_energy: 0.0
+lambda_f0: 1.0
+lambda_ph_dur: 0.3
+lambda_sent_dur: 1.0
+lambda_uv: 1.0
+lambda_word_dur: 1.0
+load_ckpt: ''
+log_interval: 100
+loud_norm: false
+lr: 0.0012
+max_beta: 0.02
+max_epochs: 3000
+max_eval_sentences: 1
+max_eval_tokens: 60000
+max_frames: 42000
+max_input_tokens: 60000
+max_sentences: 48
+max_tokens: 128000
+max_updates: 260000
+mel_loss: ssim:0.5|l1:0.5
+mel_vmax: 1.5
+mel_vmin: -6.0
+min_level_db: -120
+no_fs2: true
+norm_type: gn
+num_ckpt_keep: 10
+num_heads: 2
+num_sanity_val_steps: 1
+num_spk: 1
+num_test_samples: 0
+num_valid_plots: 10
+optimizer_adam_beta1: 0.9
+optimizer_adam_beta2: 0.98
+out_wav_norm: false
+pe_ckpt: checkpoints/0102_xiaoma_pe/model_ckpt_steps_60000.ckpt
+pe_enable: false
+perform_enhance: true
+pitch_ar: false
+pitch_enc_hidden_stride_kernel:
+- 0,2,5
+- 0,2,5
+- 0,2,5
+pitch_extractor: parselmouth
+pitch_loss: l2
+pitch_norm: log
+pitch_type: frame
+pndm_speedup: 10
+pre_align_args:
+  allow_no_txt: false
+  denoise: false
+  forced_align: mfa
+  txt_processor: zh_g2pM
+  use_sox: true
+  use_tone: false
+pre_align_cls: data_gen.singing.pre_align.SingingPreAlign
+predictor_dropout: 0.5
+predictor_grad: 0.1
+predictor_hidden: -1
+predictor_kernel: 5
+predictor_layers: 5
+prenet_dropout: 0.5
+prenet_hidden_size: 256
+pretrain_fs_ckpt: ''
+processed_data_dir: xxx
+profile_infer: false
+raw_data_dir: data/raw/raiden
+ref_norm_layer: bn
+rel_pos: true
+reset_phone_dict: true
+residual_channels: 384
+residual_layers: 20
+save_best: false
+save_ckpt: true
+save_codes:
+- configs
+- modules
+- src
+- utils
+save_f0: true
+save_gt: false
+schedule_type: linear
+seed: 1234
+sort_by_len: true
+speaker_id: raiden
+spec_max:
+- -0.4759584963321686
+- -0.04242899641394615
+- 0.2820039689540863
+- 0.6635098457336426
+- 0.7846556901931763
+- 0.9242268800735474
+- 1.0596446990966797
+- 0.9890199303627014
+- 0.8979427218437195
+- 0.8635445237159729
+- 0.8591453433036804
+- 0.6987467408180237
+- 0.717823326587677
+- 0.7350517511367798
+- 0.6464754939079285
+- 0.6164345145225525
+- 0.4986744523048401
+- 0.3543139100074768
+- 0.2876613438129425
+- 0.3467520773410797
+- 0.27083638310432434
+- 0.4002445936203003
+- 0.4222544729709625
+- 0.4776916205883026
+- 0.5299767255783081
+- 0.6194124817848206
+- 0.5802494287490845
+- 0.6222044229507446
+- 0.6124054193496704
+- 0.6688933968544006
+- 0.7368689179420471
+- 0.7275264859199524
+- 0.7448640465736389
+- 0.5364857912063599
+- 0.5867365002632141
+- 0.48127084970474243
+- 0.48270556330680847
+- 0.45863038301467896
+- 0.3647041916847229
+- 0.3207940459251404
+- 0.3352258801460266
+- 0.2846287190914154
+- 0.2674693465232849
+- 0.3205840587615967
+- 0.36074087023735046
+- 0.40593528747558594
+- 0.266417920589447
+- 0.22159256041049957
+- 0.19403372704982758
+- 0.29326388239860535
+- 0.29472747445106506
+- 0.3801038861274719
+- 0.3864395320415497
+- 0.285959392786026
+- 0.22213149070739746
+- 0.19549456238746643
+- 0.22238962352275848
+- 0.15776650607585907
+- 0.23960433900356293
+- 0.3050341308116913
+- 0.258531779050827
+- 0.19573383033275604
+- 0.2259710431098938
+- 0.2110864669084549
+- 0.24603094160556793
+- 0.05981471762061119
+- 0.17803697288036346
+- 0.17807669937610626
+- 0.18952223658561707
+- 0.053435735404491425
+- 0.1157914251089096
+- 0.026514273136854172
+- 0.16326436400413513
+- 0.22839383780956268
+- 0.08631942421197891
+- 0.23315998911857605
+- 0.162082701921463
+- 0.1533375382423401
+- 0.12668564915657043
+- 0.08644244074821472
+- 0.02113831788301468
+- 0.3463006615638733
+- 0.22695019841194153
+- 0.14443305134773254
+- 0.20211298763751984
+- 0.07295431941747665
+- -0.007622874341905117
+- -0.02703588828444481
+- -0.06394484639167786
+- -0.09371187537908554
+- -0.005024001933634281
+- -0.013857427053153515
+- -0.1372852921485901
+- -0.10361140221357346
+- -0.12665916979312897
+- -0.20724378526210785
+- -0.2142055779695511
+- 0.19261693954467773
+- 0.08877495676279068
+- -0.21178629994392395
+- -0.18947778642177582
+- -0.2520659863948822
+- -0.22880172729492188
+- -0.14105628430843353
+- -0.05572707951068878
+- -0.26297450065612793
+- -0.412873774766922
+- -0.35086822509765625
+- -0.36454305052757263
+- -0.4511951208114624
+- -0.3738810122013092
+- -0.4491288959980011
+- -0.5304299592971802
+- -0.4495029151439667
+- -0.4343602657318115
+- -0.34189438819885254
+- -0.5748119950294495
+- -0.6650391817092896
+- -0.6537092924118042
+- -0.8302493691444397
+- -0.7430973649024963
+- -0.9170511364936829
+- -1.0624077320098877
+- -1.242630958557129
+- -1.163043737411499
+- -1.0178996324539185
+- -1.4756207466125488
+- -1.5275251865386963
+spec_min:
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.999994277954102
+- -4.9844536781311035
+- -4.999994277954102
+- -4.999994277954102
+spk_cond_steps: []
+stop_token_weight: 5.0
+task_cls: training.task.SVC_task.SVCTask
+test_ids: []
+test_input_dir: ''
+test_num: 0
+test_prefixes:
+- test
+test_set_name: test
+timesteps: 1000
+train_set_name: train
+use_crepe: true
+use_denoise: false
+use_energy_embed: false
+use_gt_dur: false
+use_gt_f0: false
+use_midi: false
+use_nsf: true
+use_pitch_embed: true
+use_pos_embed: true
+use_spk_embed: false
+use_spk_id: false
+use_split_spk_id: false
+use_uv: false
+use_var_enc: false
+use_vec: false
+val_check_interval: 2000
+valid_num: 0
+valid_set_name: valid
+vocoder: network.vocoders.nsf_hifigan.NsfHifiGAN
+vocoder_ckpt: checkpoints/nsf_hifigan/model
+warmup_updates: 2000
+wav2spec_eps: 1e-6
+weight_decay: 0
+win_size: 2048
+work_dir: checkpoints/raiden

models/genshin/raiden.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1da7f6c45f8651131386c946f1bd90010b1fd568792d8d70f7025ea3b9a9dda2
+size 134965595

modules/commons/common_layers.py

@@ -0,0 +1,671 @@
+import math
+import torch
+from torch import nn
+from torch.nn import Parameter
+import torch.onnx.operators
+import torch.nn.functional as F
+import utils
+
+
+class Reshape(nn.Module):
+    def __init__(self, *args):
+        super(Reshape, self).__init__()
+        self.shape = args
+
+    def forward(self, x):
+        return x.view(self.shape)
+
+
+class Permute(nn.Module):
+    def __init__(self, *args):
+        super(Permute, self).__init__()
+        self.args = args
+
+    def forward(self, x):
+        return x.permute(self.args)
+
+
+class LinearNorm(torch.nn.Module):
+    def __init__(self, in_dim, out_dim, bias=True, w_init_gain='linear'):
+        super(LinearNorm, self).__init__()
+        self.linear_layer = torch.nn.Linear(in_dim, out_dim, bias=bias)
+
+        torch.nn.init.xavier_uniform_(
+            self.linear_layer.weight,
+            gain=torch.nn.init.calculate_gain(w_init_gain))
+
+    def forward(self, x):
+        return self.linear_layer(x)
+
+
+class ConvNorm(torch.nn.Module):
+    def __init__(self, in_channels, out_channels, kernel_size=1, stride=1,
+                 padding=None, dilation=1, bias=True, w_init_gain='linear'):
+        super(ConvNorm, self).__init__()
+        if padding is None:
+            assert (kernel_size % 2 == 1)
+            padding = int(dilation * (kernel_size - 1) / 2)
+
+        self.conv = torch.nn.Conv1d(in_channels, out_channels,
+                                    kernel_size=kernel_size, stride=stride,
+                                    padding=padding, dilation=dilation,
+                                    bias=bias)
+
+        torch.nn.init.xavier_uniform_(
+            self.conv.weight, gain=torch.nn.init.calculate_gain(w_init_gain))
+
+    def forward(self, signal):
+        conv_signal = self.conv(signal)
+        return conv_signal
+
+
+def Embedding(num_embeddings, embedding_dim, padding_idx=None):
+    m = nn.Embedding(num_embeddings, embedding_dim, padding_idx=padding_idx)
+    nn.init.normal_(m.weight, mean=0, std=embedding_dim ** -0.5)
+    if padding_idx is not None:
+        nn.init.constant_(m.weight[padding_idx], 0)
+    return m
+
+
+def LayerNorm(normalized_shape, eps=1e-5, elementwise_affine=True, export=False):
+    if not export and torch.cuda.is_available():
+        try:
+            from apex.normalization import FusedLayerNorm
+            return FusedLayerNorm(normalized_shape, eps, elementwise_affine)
+        except ImportError:
+            pass
+    return torch.nn.LayerNorm(normalized_shape, eps, elementwise_affine)
+
+
+def Linear(in_features, out_features, bias=True):
+    m = nn.Linear(in_features, out_features, bias)
+    nn.init.xavier_uniform_(m.weight)
+    if bias:
+        nn.init.constant_(m.bias, 0.)
+    return m
+
+
+class SinusoidalPositionalEmbedding(nn.Module):
+    """This module produces sinusoidal positional embeddings of any length.
+
+    Padding symbols are ignored.
+    """
+
+    def __init__(self, embedding_dim, padding_idx, init_size=1024):
+        super().__init__()
+        self.embedding_dim = embedding_dim
+        self.padding_idx = padding_idx
+        self.weights = SinusoidalPositionalEmbedding.get_embedding(
+            init_size,
+            embedding_dim,
+            padding_idx,
+        )
+        self.register_buffer('_float_tensor', torch.FloatTensor(1))
+
+    @staticmethod
+    def get_embedding(num_embeddings, embedding_dim, padding_idx=None):
+        """Build sinusoidal embeddings.
+
+        This matches the implementation in tensor2tensor, but differs slightly
+        from the description in Section 3.5 of "Attention Is All You Need".
+        """
+        half_dim = embedding_dim // 2
+        emb = math.log(10000) / (half_dim - 1)
+        emb = torch.exp(torch.arange(half_dim, dtype=torch.float) * -emb)
+        emb = torch.arange(num_embeddings, dtype=torch.float).unsqueeze(1) * emb.unsqueeze(0)
+        emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1).view(num_embeddings, -1)
+        if embedding_dim % 2 == 1:
+            # zero pad
+            emb = torch.cat([emb, torch.zeros(num_embeddings, 1)], dim=1)
+        if padding_idx is not None:
+            emb[padding_idx, :] = 0
+        return emb
+
+    def forward(self, input, incremental_state=None, timestep=None, positions=None, **kwargs):
+        """Input is expected to be of size [bsz x seqlen]."""
+        bsz, seq_len = input.shape[:2]
+        max_pos = self.padding_idx + 1 + seq_len
+        if self.weights is None or max_pos > self.weights.size(0):
+            # recompute/expand embeddings if needed
+            self.weights = SinusoidalPositionalEmbedding.get_embedding(
+                max_pos,
+                self.embedding_dim,
+                self.padding_idx,
+            )
+        self.weights = self.weights.to(self._float_tensor)
+
+        if incremental_state is not None:
+            # positions is the same for every token when decoding a single step
+            pos = timestep.view(-1)[0] + 1 if timestep is not None else seq_len
+            return self.weights[self.padding_idx + pos, :].expand(bsz, 1, -1)
+
+        positions = utils.make_positions(input, self.padding_idx) if positions is None else positions
+        return self.weights.index_select(0, positions.view(-1)).view(bsz, seq_len, -1).detach()
+
+    def max_positions(self):
+        """Maximum number of supported positions."""
+        return int(1e5)  # an arbitrary large number
+
+
+class ConvTBC(nn.Module):
+    def __init__(self, in_channels, out_channels, kernel_size, padding=0):
+        super(ConvTBC, self).__init__()
+        self.in_channels = in_channels
+        self.out_channels = out_channels
+        self.kernel_size = kernel_size
+        self.padding = padding
+
+        self.weight = torch.nn.Parameter(torch.Tensor(
+            self.kernel_size, in_channels, out_channels))
+        self.bias = torch.nn.Parameter(torch.Tensor(out_channels))
+
+    def forward(self, input):
+        return torch.conv_tbc(input.contiguous(), self.weight, self.bias, self.padding)
+
+
+class MultiheadAttention(nn.Module):
+    def __init__(self, embed_dim, num_heads, kdim=None, vdim=None, dropout=0., bias=True,
+                 add_bias_kv=False, add_zero_attn=False, self_attention=False,
+                 encoder_decoder_attention=False):
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.kdim = kdim if kdim is not None else embed_dim
+        self.vdim = vdim if vdim is not None else embed_dim
+        self.qkv_same_dim = self.kdim == embed_dim and self.vdim == embed_dim
+
+        self.num_heads = num_heads
+        self.dropout = dropout
+        self.head_dim = embed_dim // num_heads
+        assert self.head_dim * num_heads == self.embed_dim, "embed_dim must be divisible by num_heads"
+        self.scaling = self.head_dim ** -0.5
+
+        self.self_attention = self_attention
+        self.encoder_decoder_attention = encoder_decoder_attention
+
+        assert not self.self_attention or self.qkv_same_dim, 'Self-attention requires query, key and ' \
+                                                             'value to be of the same size'
+
+        if self.qkv_same_dim:
+            self.in_proj_weight = Parameter(torch.Tensor(3 * embed_dim, embed_dim))
+        else:
+            self.k_proj_weight = Parameter(torch.Tensor(embed_dim, self.kdim))
+            self.v_proj_weight = Parameter(torch.Tensor(embed_dim, self.vdim))
+            self.q_proj_weight = Parameter(torch.Tensor(embed_dim, embed_dim))
+
+        if bias:
+            self.in_proj_bias = Parameter(torch.Tensor(3 * embed_dim))
+        else:
+            self.register_parameter('in_proj_bias', None)
+
+        self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)
+
+        if add_bias_kv:
+            self.bias_k = Parameter(torch.Tensor(1, 1, embed_dim))
+            self.bias_v = Parameter(torch.Tensor(1, 1, embed_dim))
+        else:
+            self.bias_k = self.bias_v = None
+
+        self.add_zero_attn = add_zero_attn
+
+        self.reset_parameters()
+
+        self.enable_torch_version = False
+        if hasattr(F, "multi_head_attention_forward"):
+            self.enable_torch_version = True
+        else:
+            self.enable_torch_version = False
+        self.last_attn_probs = None
+
+    def reset_parameters(self):
+        if self.qkv_same_dim:
+            nn.init.xavier_uniform_(self.in_proj_weight)
+        else:
+            nn.init.xavier_uniform_(self.k_proj_weight)
+            nn.init.xavier_uniform_(self.v_proj_weight)
+            nn.init.xavier_uniform_(self.q_proj_weight)
+
+        nn.init.xavier_uniform_(self.out_proj.weight)
+        if self.in_proj_bias is not None:
+            nn.init.constant_(self.in_proj_bias, 0.)
+            nn.init.constant_(self.out_proj.bias, 0.)
+        if self.bias_k is not None:
+            nn.init.xavier_normal_(self.bias_k)
+        if self.bias_v is not None:
+            nn.init.xavier_normal_(self.bias_v)
+
+    def forward(
+            self,
+            query, key, value,
+            key_padding_mask=None,
+            incremental_state=None,
+            need_weights=True,
+            static_kv=False,
+            attn_mask=None,
+            before_softmax=False,
+            need_head_weights=False,
+            enc_dec_attn_constraint_mask=None,
+            reset_attn_weight=None
+    ):
+        """Input shape: Time x Batch x Channel
+
+        Args:
+            key_padding_mask (ByteTensor, optional): mask to exclude
+                keys that are pads, of shape `(batch, src_len)`, where
+                padding elements are indicated by 1s.
+            need_weights (bool, optional): return the attention weights,
+                averaged over heads (default: False).
+            attn_mask (ByteTensor, optional): typically used to
+                implement causal attention, where the mask prevents the
+                attention from looking forward in time (default: None).
+            before_softmax (bool, optional): return the raw attention
+                weights and values before the attention softmax.
+            need_head_weights (bool, optional): return the attention
+                weights for each head. Implies *need_weights*. Default:
+                return the average attention weights over all heads.
+        """
+        if need_head_weights:
+            need_weights = True
+
+        tgt_len, bsz, embed_dim = query.size()
+        assert embed_dim == self.embed_dim
+        assert list(query.size()) == [tgt_len, bsz, embed_dim]
+
+        if self.enable_torch_version and incremental_state is None and not static_kv and reset_attn_weight is None:
+            if self.qkv_same_dim:
+                return F.multi_head_attention_forward(query, key, value,
+                                                      self.embed_dim, self.num_heads,
+                                                      self.in_proj_weight,
+                                                      self.in_proj_bias, self.bias_k, self.bias_v,
+                                                      self.add_zero_attn, self.dropout,
+                                                      self.out_proj.weight, self.out_proj.bias,
+                                                      self.training, key_padding_mask, need_weights,
+                                                      attn_mask)
+            else:
+                return F.multi_head_attention_forward(query, key, value,
+                                                      self.embed_dim, self.num_heads,
+                                                      torch.empty([0]),
+                                                      self.in_proj_bias, self.bias_k, self.bias_v,
+                                                      self.add_zero_attn, self.dropout,
+                                                      self.out_proj.weight, self.out_proj.bias,
+                                                      self.training, key_padding_mask, need_weights,
+                                                      attn_mask, use_separate_proj_weight=True,
+                                                      q_proj_weight=self.q_proj_weight,
+                                                      k_proj_weight=self.k_proj_weight,
+                                                      v_proj_weight=self.v_proj_weight)
+
+        if incremental_state is not None:
+            print('Not implemented error.')
+            exit()
+        else:
+            saved_state = None
+
+        if self.self_attention:
+            # self-attention
+            q, k, v = self.in_proj_qkv(query)
+        elif self.encoder_decoder_attention:
+            # encoder-decoder attention
+            q = self.in_proj_q(query)
+            if key is None:
+                assert value is None
+                k = v = None
+            else:
+                k = self.in_proj_k(key)
+                v = self.in_proj_v(key)
+
+        else:
+            q = self.in_proj_q(query)
+            k = self.in_proj_k(key)
+            v = self.in_proj_v(value)
+        q *= self.scaling
+
+        if self.bias_k is not None:
+            assert self.bias_v is not None
+            k = torch.cat([k, self.bias_k.repeat(1, bsz, 1)])
+            v = torch.cat([v, self.bias_v.repeat(1, bsz, 1)])
+            if attn_mask is not None:
+                attn_mask = torch.cat([attn_mask, attn_mask.new_zeros(attn_mask.size(0), 1)], dim=1)
+            if key_padding_mask is not None:
+                key_padding_mask = torch.cat(
+                    [key_padding_mask, key_padding_mask.new_zeros(key_padding_mask.size(0), 1)], dim=1)
+
+        q = q.contiguous().view(tgt_len, bsz * self.num_heads, self.head_dim).transpose(0, 1)
+        if k is not None:
+            k = k.contiguous().view(-1, bsz * self.num_heads, self.head_dim).transpose(0, 1)
+        if v is not None:
+            v = v.contiguous().view(-1, bsz * self.num_heads, self.head_dim).transpose(0, 1)
+
+        if saved_state is not None:
+            print('Not implemented error.')
+            exit()
+
+        src_len = k.size(1)
+
+        # This is part of a workaround to get around fork/join parallelism
+        # not supporting Optional types.
+        if key_padding_mask is not None and key_padding_mask.shape == torch.Size([]):
+            key_padding_mask = None
+
+        if key_padding_mask is not None:
+            assert key_padding_mask.size(0) == bsz
+            assert key_padding_mask.size(1) == src_len
+
+        if self.add_zero_attn:
+            src_len += 1
+            k = torch.cat([k, k.new_zeros((k.size(0), 1) + k.size()[2:])], dim=1)
+            v = torch.cat([v, v.new_zeros((v.size(0), 1) + v.size()[2:])], dim=1)
+            if attn_mask is not None:
+                attn_mask = torch.cat([attn_mask, attn_mask.new_zeros(attn_mask.size(0), 1)], dim=1)
+            if key_padding_mask is not None:
+                key_padding_mask = torch.cat(
+                    [key_padding_mask, torch.zeros(key_padding_mask.size(0), 1).type_as(key_padding_mask)], dim=1)
+
+        attn_weights = torch.bmm(q, k.transpose(1, 2))
+        attn_weights = self.apply_sparse_mask(attn_weights, tgt_len, src_len, bsz)
+
+        assert list(attn_weights.size()) == [bsz * self.num_heads, tgt_len, src_len]
+
+        if attn_mask is not None:
+            if len(attn_mask.shape) == 2:
+                attn_mask = attn_mask.unsqueeze(0)
+            elif len(attn_mask.shape) == 3:
+                attn_mask = attn_mask[:, None].repeat([1, self.num_heads, 1, 1]).reshape(
+                    bsz * self.num_heads, tgt_len, src_len)
+            attn_weights = attn_weights + attn_mask
+
+        if enc_dec_attn_constraint_mask is not None:  # bs x head x L_kv
+            attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
+            attn_weights = attn_weights.masked_fill(
+                enc_dec_attn_constraint_mask.unsqueeze(2).bool(),
+                -1e9,
+            )
+            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
+
+        if key_padding_mask is not None:
+            # don't attend to padding symbols
+            attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
+            attn_weights = attn_weights.masked_fill(
+                key_padding_mask.unsqueeze(1).unsqueeze(2),
+                -1e9,
+            )
+            attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)
+
+        attn_logits = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)
+
+        if before_softmax:
+            return attn_weights, v
+
+        attn_weights_float = utils.softmax(attn_weights, dim=-1)
+        attn_weights = attn_weights_float.type_as(attn_weights)
+        attn_probs = F.dropout(attn_weights_float.type_as(attn_weights), p=self.dropout, training=self.training)
+
+        if reset_attn_weight is not None:
+            if reset_attn_weight:
+                self.last_attn_probs = attn_probs.detach()
+            else:
+                assert self.last_attn_probs is not None
+                attn_probs = self.last_attn_probs
+        attn = torch.bmm(attn_probs, v)
+        assert list(attn.size()) == [bsz * self.num_heads, tgt_len, self.head_dim]
+        attn = attn.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim)
+        attn = self.out_proj(attn)
+
+        if need_weights:
+            attn_weights = attn_weights_float.view(bsz, self.num_heads, tgt_len, src_len).transpose(1, 0)
+            if not need_head_weights:
+                # average attention weights over heads
+                attn_weights = attn_weights.mean(dim=0)
+        else:
+            attn_weights = None
+
+        return attn, (attn_weights, attn_logits)
+
+    def in_proj_qkv(self, query):
+        return self._in_proj(query).chunk(3, dim=-1)
+
+    def in_proj_q(self, query):
+        if self.qkv_same_dim:
+            return self._in_proj(query, end=self.embed_dim)
+        else:
+            bias = self.in_proj_bias
+            if bias is not None:
+                bias = bias[:self.embed_dim]
+            return F.linear(query, self.q_proj_weight, bias)
+
+    def in_proj_k(self, key):
+        if self.qkv_same_dim:
+            return self._in_proj(key, start=self.embed_dim, end=2 * self.embed_dim)
+        else:
+            weight = self.k_proj_weight
+            bias = self.in_proj_bias
+            if bias is not None:
+                bias = bias[self.embed_dim:2 * self.embed_dim]
+            return F.linear(key, weight, bias)
+
+    def in_proj_v(self, value):
+        if self.qkv_same_dim:
+            return self._in_proj(value, start=2 * self.embed_dim)
+        else:
+            weight = self.v_proj_weight
+            bias = self.in_proj_bias
+            if bias is not None:
+                bias = bias[2 * self.embed_dim:]
+            return F.linear(value, weight, bias)
+
+    def _in_proj(self, input, start=0, end=None):
+        weight = self.in_proj_weight
+        bias = self.in_proj_bias
+        weight = weight[start:end, :]
+        if bias is not None:
+            bias = bias[start:end]
+        return F.linear(input, weight, bias)
+
+
+    def apply_sparse_mask(self, attn_weights, tgt_len, src_len, bsz):
+        return attn_weights
+
+
+class Swish(torch.autograd.Function):
+    @staticmethod
+    def forward(ctx, i):
+        result = i * torch.sigmoid(i)
+        ctx.save_for_backward(i)
+        return result
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        i = ctx.saved_variables[0]
+        sigmoid_i = torch.sigmoid(i)
+        return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i)))
+
+
+class CustomSwish(nn.Module):
+    def forward(self, input_tensor):
+        return Swish.apply(input_tensor)
+        
+class Mish(nn.Module):
+    def forward(self, x):
+        return x * torch.tanh(F.softplus(x))
+
+class TransformerFFNLayer(nn.Module):
+    def __init__(self, hidden_size, filter_size, padding="SAME", kernel_size=1, dropout=0., act='gelu'):
+        super().__init__()
+        self.kernel_size = kernel_size
+        self.dropout = dropout
+        self.act = act
+        if padding == 'SAME':
+            self.ffn_1 = nn.Conv1d(hidden_size, filter_size, kernel_size, padding=kernel_size // 2)
+        elif padding == 'LEFT':
+            self.ffn_1 = nn.Sequential(
+                nn.ConstantPad1d((kernel_size - 1, 0), 0.0),
+                nn.Conv1d(hidden_size, filter_size, kernel_size)
+            )
+        self.ffn_2 = Linear(filter_size, hidden_size)
+        if self.act == 'swish':
+            self.swish_fn = CustomSwish()
+
+    def forward(self, x, incremental_state=None):
+        # x: T x B x C
+        if incremental_state is not None:
+            assert incremental_state is None, 'Nar-generation does not allow this.'
+            exit(1)
+
+        x = self.ffn_1(x.permute(1, 2, 0)).permute(2, 0, 1)
+        x = x * self.kernel_size ** -0.5
+
+        if incremental_state is not None:
+            x = x[-1:]
+        if self.act == 'gelu':
+            x = F.gelu(x)
+        if self.act == 'relu':
+            x = F.relu(x)
+        if self.act == 'swish':
+            x = self.swish_fn(x)
+        x = F.dropout(x, self.dropout, training=self.training)
+        x = self.ffn_2(x)
+        return x
+
+
+class BatchNorm1dTBC(nn.Module):
+    def __init__(self, c):
+        super(BatchNorm1dTBC, self).__init__()
+        self.bn = nn.BatchNorm1d(c)
+
+    def forward(self, x):
+        """
+
+        :param x: [T, B, C]
+        :return: [T, B, C]
+        """
+        x = x.permute(1, 2, 0)  # [B, C, T]
+        x = self.bn(x)  # [B, C, T]
+        x = x.permute(2, 0, 1)  # [T, B, C]
+        return x
+
+
+class EncSALayer(nn.Module):
+    def __init__(self, c, num_heads, dropout, attention_dropout=0.1,
+                 relu_dropout=0.1, kernel_size=9, padding='SAME', norm='ln', act='gelu'):
+        super().__init__()
+        self.c = c
+        self.dropout = dropout
+        self.num_heads = num_heads
+        if num_heads > 0:
+            if norm == 'ln':
+                self.layer_norm1 = LayerNorm(c)
+            elif norm == 'bn':
+                self.layer_norm1 = BatchNorm1dTBC(c)
+            self.self_attn = MultiheadAttention(
+                self.c, num_heads, self_attention=True, dropout=attention_dropout, bias=False,
+            )
+        if norm == 'ln':
+            self.layer_norm2 = LayerNorm(c)
+        elif norm == 'bn':
+            self.layer_norm2 = BatchNorm1dTBC(c)
+        self.ffn = TransformerFFNLayer(
+            c, 4 * c, kernel_size=kernel_size, dropout=relu_dropout, padding=padding, act=act)
+
+    def forward(self, x, encoder_padding_mask=None, **kwargs):
+        layer_norm_training = kwargs.get('layer_norm_training', None)
+        if layer_norm_training is not None:
+            self.layer_norm1.training = layer_norm_training
+            self.layer_norm2.training = layer_norm_training
+        if self.num_heads > 0:
+            residual = x
+            x = self.layer_norm1(x)
+            x, _, = self.self_attn(
+                query=x,
+                key=x,
+                value=x,
+                key_padding_mask=encoder_padding_mask
+            )
+            x = F.dropout(x, self.dropout, training=self.training)
+            x = residual + x
+            x = x * (1 - encoder_padding_mask.float()).transpose(0, 1)[..., None]
+
+        residual = x
+        x = self.layer_norm2(x)
+        x = self.ffn(x)
+        x = F.dropout(x, self.dropout, training=self.training)
+        x = residual + x
+        x = x * (1 - encoder_padding_mask.float()).transpose(0, 1)[..., None]
+        return x
+
+
+class DecSALayer(nn.Module):
+    def __init__(self, c, num_heads, dropout, attention_dropout=0.1, relu_dropout=0.1, kernel_size=9, act='gelu'):
+        super().__init__()
+        self.c = c
+        self.dropout = dropout
+        self.layer_norm1 = LayerNorm(c)
+        self.self_attn = MultiheadAttention(
+            c, num_heads, self_attention=True, dropout=attention_dropout, bias=False
+        )
+        self.layer_norm2 = LayerNorm(c)
+        self.encoder_attn = MultiheadAttention(
+            c, num_heads, encoder_decoder_attention=True, dropout=attention_dropout, bias=False,
+        )
+        self.layer_norm3 = LayerNorm(c)
+        self.ffn = TransformerFFNLayer(
+            c, 4 * c, padding='LEFT', kernel_size=kernel_size, dropout=relu_dropout, act=act)
+
+    def forward(
+            self,
+            x,
+            encoder_out=None,
+            encoder_padding_mask=None,
+            incremental_state=None,
+            self_attn_mask=None,
+            self_attn_padding_mask=None,
+            attn_out=None,
+            reset_attn_weight=None,
+            **kwargs,
+    ):
+        layer_norm_training = kwargs.get('layer_norm_training', None)
+        if layer_norm_training is not None:
+            self.layer_norm1.training = layer_norm_training
+            self.layer_norm2.training = layer_norm_training
+            self.layer_norm3.training = layer_norm_training
+        residual = x
+        x = self.layer_norm1(x)
+        x, _ = self.self_attn(
+            query=x,
+            key=x,
+            value=x,
+            key_padding_mask=self_attn_padding_mask,
+            incremental_state=incremental_state,
+            attn_mask=self_attn_mask
+        )
+        x = F.dropout(x, self.dropout, training=self.training)
+        x = residual + x
+
+        residual = x
+        x = self.layer_norm2(x)
+        if encoder_out is not None:
+            x, attn = self.encoder_attn(
+                query=x,
+                key=encoder_out,
+                value=encoder_out,
+                key_padding_mask=encoder_padding_mask,
+                incremental_state=incremental_state,
+                static_kv=True,
+                enc_dec_attn_constraint_mask=None, #utils.get_incremental_state(self, incremental_state, 'enc_dec_attn_constraint_mask'),
+                reset_attn_weight=reset_attn_weight
+            )
+            attn_logits = attn[1]
+        else:
+            assert attn_out is not None
+            x = self.encoder_attn.in_proj_v(attn_out.transpose(0, 1))
+            attn_logits = None
+        x = F.dropout(x, self.dropout, training=self.training)
+        x = residual + x
+
+        residual = x
+        x = self.layer_norm3(x)
+        x = self.ffn(x, incremental_state=incremental_state)
+        x = F.dropout(x, self.dropout, training=self.training)
+        x = residual + x
+        # if len(attn_logits.size()) > 3:
+        #    indices = attn_logits.softmax(-1).max(-1).values.sum(-1).argmax(-1)
+        #    attn_logits = attn_logits.gather(1,
+        #        indices[:, None, None, None].repeat(1, 1, attn_logits.size(-2), attn_logits.size(-1))).squeeze(1)
+        return x, attn_logits

modules/commons/espnet_positional_embedding.py

@@ -0,0 +1,113 @@
+import math
+import torch
+
+
+class PositionalEncoding(torch.nn.Module):
+    """Positional encoding.
+    Args:
+        d_model (int): Embedding dimension.
+        dropout_rate (float): Dropout rate.
+        max_len (int): Maximum input length.
+        reverse (bool): Whether to reverse the input position.
+    """
+
+    def __init__(self, d_model, dropout_rate, max_len=5000, reverse=False):
+        """Construct an PositionalEncoding object."""
+        super(PositionalEncoding, self).__init__()
+        self.d_model = d_model
+        self.reverse = reverse
+        self.xscale = math.sqrt(self.d_model)
+        self.dropout = torch.nn.Dropout(p=dropout_rate)
+        self.pe = None
+        self.extend_pe(torch.tensor(0.0).expand(1, max_len))
+
+    def extend_pe(self, x):
+        """Reset the positional encodings."""
+        if self.pe is not None:
+            if self.pe.size(1) >= x.size(1):
+                if self.pe.dtype != x.dtype or self.pe.device != x.device:
+                    self.pe = self.pe.to(dtype=x.dtype, device=x.device)
+                return
+        pe = torch.zeros(x.size(1), self.d_model)
+        if self.reverse:
+            position = torch.arange(
+                x.size(1) - 1, -1, -1.0, dtype=torch.float32
+            ).unsqueeze(1)
+        else:
+            position = torch.arange(0, x.size(1), dtype=torch.float32).unsqueeze(1)
+        div_term = torch.exp(
+            torch.arange(0, self.d_model, 2, dtype=torch.float32)
+            * -(math.log(10000.0) / self.d_model)
+        )
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+        pe = pe.unsqueeze(0)
+        self.pe = pe.to(device=x.device, dtype=x.dtype)
+
+    def forward(self, x: torch.Tensor):
+        """Add positional encoding.
+        Args:
+            x (torch.Tensor): Input tensor (batch, time, `*`).
+        Returns:
+            torch.Tensor: Encoded tensor (batch, time, `*`).
+        """
+        self.extend_pe(x)
+        x = x * self.xscale + self.pe[:, : x.size(1)]
+        return self.dropout(x)
+
+
+class ScaledPositionalEncoding(PositionalEncoding):
+    """Scaled positional encoding module.
+    See Sec. 3.2  https://arxiv.org/abs/1809.08895
+    Args:
+        d_model (int): Embedding dimension.
+        dropout_rate (float): Dropout rate.
+        max_len (int): Maximum input length.
+    """
+
+    def __init__(self, d_model, dropout_rate, max_len=5000):
+        """Initialize class."""
+        super().__init__(d_model=d_model, dropout_rate=dropout_rate, max_len=max_len)
+        self.alpha = torch.nn.Parameter(torch.tensor(1.0))
+
+    def reset_parameters(self):
+        """Reset parameters."""
+        self.alpha.data = torch.tensor(1.0)
+
+    def forward(self, x):
+        """Add positional encoding.
+        Args:
+            x (torch.Tensor): Input tensor (batch, time, `*`).
+        Returns:
+            torch.Tensor: Encoded tensor (batch, time, `*`).
+        """
+        self.extend_pe(x)
+        x = x + self.alpha * self.pe[:, : x.size(1)]
+        return self.dropout(x)
+
+
+class RelPositionalEncoding(PositionalEncoding):
+    """Relative positional encoding module.
+    See : Appendix B in https://arxiv.org/abs/1901.02860
+    Args:
+        d_model (int): Embedding dimension.
+        dropout_rate (float): Dropout rate.
+        max_len (int): Maximum input length.
+    """
+
+    def __init__(self, d_model, dropout_rate, max_len=5000):
+        """Initialize class."""
+        super().__init__(d_model, dropout_rate, max_len, reverse=True)
+
+    def forward(self, x):
+        """Compute positional encoding.
+        Args:
+            x (torch.Tensor): Input tensor (batch, time, `*`).
+        Returns:
+            torch.Tensor: Encoded tensor (batch, time, `*`).
+            torch.Tensor: Positional embedding tensor (1, time, `*`).
+        """
+        self.extend_pe(x)
+        x = x * self.xscale
+        pos_emb = self.pe[:, : x.size(1)]
+        return self.dropout(x) + self.dropout(pos_emb)

modules/commons/ssim.py

@@ -0,0 +1,391 @@
+# '''
+# https://github.com/One-sixth/ms_ssim_pytorch/blob/master/ssim.py
+# '''
+#
+# import torch
+# import torch.jit
+# import torch.nn.functional as F
+#
+#
+# @torch.jit.script
+# def create_window(window_size: int, sigma: float, channel: int):
+#     '''
+#     Create 1-D gauss kernel
+#     :param window_size: the size of gauss kernel
+#     :param sigma: sigma of normal distribution
+#     :param channel: input channel
+#     :return: 1D kernel
+#     '''
+#     coords = torch.arange(window_size, dtype=torch.float)
+#     coords -= window_size // 2
+#
+#     g = torch.exp(-(coords ** 2) / (2 * sigma ** 2))
+#     g /= g.sum()
+#
+#     g = g.reshape(1, 1, 1, -1).repeat(channel, 1, 1, 1)
+#     return g
+#
+#
+# @torch.jit.script
+# def _gaussian_filter(x, window_1d, use_padding: bool):
+#     '''
+#     Blur input with 1-D kernel
+#     :param x: batch of tensors to be blured
+#     :param window_1d: 1-D gauss kernel
+#     :param use_padding: padding image before conv
+#     :return: blured tensors
+#     '''
+#     C = x.shape[1]
+#     padding = 0
+#     if use_padding:
+#         window_size = window_1d.shape[3]
+#         padding = window_size // 2
+#     out = F.conv2d(x, window_1d, stride=1, padding=(0, padding), groups=C)
+#     out = F.conv2d(out, window_1d.transpose(2, 3), stride=1, padding=(padding, 0), groups=C)
+#     return out
+#
+#
+# @torch.jit.script
+# def ssim(X, Y, window, data_range: float, use_padding: bool = False):
+#     '''
+#     Calculate ssim index for X and Y
+#     :param X: images [B, C, H, N_bins]
+#     :param Y: images [B, C, H, N_bins]
+#     :param window: 1-D gauss kernel
+#     :param data_range: value range of input images. (usually 1.0 or 255)
+#     :param use_padding: padding image before conv
+#     :return:
+#     '''
+#
+#     K1 = 0.01
+#     K2 = 0.03
+#     compensation = 1.0
+#
+#     C1 = (K1 * data_range) ** 2
+#     C2 = (K2 * data_range) ** 2
+#
+#     mu1 = _gaussian_filter(X, window, use_padding)
+#     mu2 = _gaussian_filter(Y, window, use_padding)
+#     sigma1_sq = _gaussian_filter(X * X, window, use_padding)
+#     sigma2_sq = _gaussian_filter(Y * Y, window, use_padding)
+#     sigma12 = _gaussian_filter(X * Y, window, use_padding)
+#
+#     mu1_sq = mu1.pow(2)
+#     mu2_sq = mu2.pow(2)
+#     mu1_mu2 = mu1 * mu2
+#
+#     sigma1_sq = compensation * (sigma1_sq - mu1_sq)
+#     sigma2_sq = compensation * (sigma2_sq - mu2_sq)
+#     sigma12 = compensation * (sigma12 - mu1_mu2)
+#
+#     cs_map = (2 * sigma12 + C2) / (sigma1_sq + sigma2_sq + C2)
+#     # Fixed the issue that the negative value of cs_map caused ms_ssim to output Nan.
+#     cs_map = cs_map.clamp_min(0.)
+#     ssim_map = ((2 * mu1_mu2 + C1) / (mu1_sq + mu2_sq + C1)) * cs_map
+#
+#     ssim_val = ssim_map.mean(dim=(1, 2, 3))  # reduce along CHW
+#     cs = cs_map.mean(dim=(1, 2, 3))
+#
+#     return ssim_val, cs
+#
+#
+# @torch.jit.script
+# def ms_ssim(X, Y, window, data_range: float, weights, use_padding: bool = False, eps: float = 1e-8):
+#     '''
+#     interface of ms-ssim
+#     :param X: a batch of images, (N,C,H,W)
+#     :param Y: a batch of images, (N,C,H,W)
+#     :param window: 1-D gauss kernel
+#     :param data_range: value range of input images. (usually 1.0 or 255)
+#     :param weights: weights for different levels
+#     :param use_padding: padding image before conv
+#     :param eps: use for avoid grad nan.
+#     :return:
+#     '''
+#     levels = weights.shape[0]
+#     cs_vals = []
+#     ssim_vals = []
+#     for _ in range(levels):
+#         ssim_val, cs = ssim(X, Y, window=window, data_range=data_range, use_padding=use_padding)
+#         # Use for fix a issue. When c = a ** b and a is 0, c.backward() will cause the a.grad become inf.
+#         ssim_val = ssim_val.clamp_min(eps)
+#         cs = cs.clamp_min(eps)
+#         cs_vals.append(cs)
+#
+#         ssim_vals.append(ssim_val)
+#         padding = (X.shape[2] % 2, X.shape[3] % 2)
+#         X = F.avg_pool2d(X, kernel_size=2, stride=2, padding=padding)
+#         Y = F.avg_pool2d(Y, kernel_size=2, stride=2, padding=padding)
+#
+#     cs_vals = torch.stack(cs_vals, dim=0)
+#     ms_ssim_val = torch.prod((cs_vals[:-1] ** weights[:-1].unsqueeze(1)) * (ssim_vals[-1] ** weights[-1]), dim=0)
+#     return ms_ssim_val
+#
+#
+# class SSIM(torch.jit.ScriptModule):
+#     __constants__ = ['data_range', 'use_padding']
+#
+#     def __init__(self, window_size=11, window_sigma=1.5, data_range=255., channel=3, use_padding=False):
+#         '''
+#         :param window_size: the size of gauss kernel
+#         :param window_sigma: sigma of normal distribution
+#         :param data_range: value range of input images. (usually 1.0 or 255)
+#         :param channel: input channels (default: 3)
+#         :param use_padding: padding image before conv
+#         '''
+#         super().__init__()
+#         assert window_size % 2 == 1, 'Window size must be odd.'
+#         window = create_window(window_size, window_sigma, channel)
+#         self.register_buffer('window', window)
+#         self.data_range = data_range
+#         self.use_padding = use_padding
+#
+#     @torch.jit.script_method
+#     def forward(self, X, Y):
+#         r = ssim(X, Y, window=self.window, data_range=self.data_range, use_padding=self.use_padding)
+#         return r[0]
+#
+#
+# class MS_SSIM(torch.jit.ScriptModule):
+#     __constants__ = ['data_range', 'use_padding', 'eps']
+#
+#     def __init__(self, window_size=11, window_sigma=1.5, data_range=255., channel=3, use_padding=False, weights=None,
+#                  levels=None, eps=1e-8):
+#         '''
+#         class for ms-ssim
+#         :param window_size: the size of gauss kernel
+#         :param window_sigma: sigma of normal distribution
+#         :param data_range: value range of input images. (usually 1.0 or 255)
+#         :param channel: input channels
+#         :param use_padding: padding image before conv
+#         :param weights: weights for different levels. (default [0.0448, 0.2856, 0.3001, 0.2363, 0.1333])
+#         :param levels: number of downsampling
+#         :param eps: Use for fix a issue. When c = a ** b and a is 0, c.backward() will cause the a.grad become inf.
+#         '''
+#         super().__init__()
+#         assert window_size % 2 == 1, 'Window size must be odd.'
+#         self.data_range = data_range
+#         self.use_padding = use_padding
+#         self.eps = eps
+#
+#         window = create_window(window_size, window_sigma, channel)
+#         self.register_buffer('window', window)
+#
+#         if weights is None:
+#             weights = [0.0448, 0.2856, 0.3001, 0.2363, 0.1333]
+#         weights = torch.tensor(weights, dtype=torch.float)
+#
+#         if levels is not None:
+#             weights = weights[:levels]
+#             weights = weights / weights.sum()
+#
+#         self.register_buffer('weights', weights)
+#
+#     @torch.jit.script_method
+#     def forward(self, X, Y):
+#         return ms_ssim(X, Y, window=self.window, data_range=self.data_range, weights=self.weights,
+#                        use_padding=self.use_padding, eps=self.eps)
+#
+#
+# if __name__ == '__main__':
+#     print('Simple Test')
+#     im = torch.randint(0, 255, (5, 3, 256, 256), dtype=torch.float, device='cuda')
+#     img1 = im / 255
+#     img2 = img1 * 0.5
+#
+#     losser = SSIM(data_range=1.).cuda()
+#     loss = losser(img1, img2).mean()
+#
+#     losser2 = MS_SSIM(data_range=1.).cuda()
+#     loss2 = losser2(img1, img2).mean()
+#
+#     print(loss.item())
+#     print(loss2.item())
+#
+# if __name__ == '__main__':
+#     print('Training Test')
+#     import cv2
+#     import torch.optim
+#     import numpy as np
+#     import imageio
+#     import time
+#
+#     out_test_video = False
+#     # 最好不要直接输出gif图，会非常大，最好先输出mkv文件后用ffmpeg转换到GIF
+#     video_use_gif = False
+#
+#     im = cv2.imread('test_img1.jpg', 1)
+#     t_im = torch.from_numpy(im).cuda().permute(2, 0, 1).float()[None] / 255.
+#
+#     if out_test_video:
+#         if video_use_gif:
+#             fps = 0.5
+#             out_wh = (im.shape[1] // 2, im.shape[0] // 2)
+#             suffix = '.gif'
+#         else:
+#             fps = 5
+#             out_wh = (im.shape[1], im.shape[0])
+#             suffix = '.mkv'
+#         video_last_time = time.perf_counter()
+#         video = imageio.get_writer('ssim_test' + suffix, fps=fps)
+#
+#     # 测试ssim
+#     print('Training SSIM')
+#     rand_im = torch.randint_like(t_im, 0, 255, dtype=torch.float32) / 255.
+#     rand_im.requires_grad = True
+#     optim = torch.optim.Adam([rand_im], 0.003, eps=1e-8)
+#     losser = SSIM(data_range=1., channel=t_im.shape[1]).cuda()
+#     ssim_score = 0
+#     while ssim_score < 0.999:
+#         optim.zero_grad()
+#         loss = losser(rand_im, t_im)
+#         (-loss).sum().backward()
+#         ssim_score = loss.item()
+#         optim.step()
+#         r_im = np.transpose(rand_im.detach().cpu().numpy().clip(0, 1) * 255, [0, 2, 3, 1]).astype(np.uint8)[0]
+#         r_im = cv2.putText(r_im, 'ssim %f' % ssim_score, (10, 30), cv2.FONT_HERSHEY_PLAIN, 2, (255, 0, 0), 2)
+#
+#         if out_test_video:
+#             if time.perf_counter() - video_last_time > 1. / fps:
+#                 video_last_time = time.perf_counter()
+#                 out_frame = cv2.cvtColor(r_im, cv2.COLOR_BGR2RGB)
+#                 out_frame = cv2.resize(out_frame, out_wh, interpolation=cv2.INTER_AREA)
+#                 if isinstance(out_frame, cv2.UMat):
+#                     out_frame = out_frame.get()
+#                 video.append_data(out_frame)
+#
+#         cv2.imshow('ssim', r_im)
+#         cv2.setWindowTitle('ssim', 'ssim %f' % ssim_score)
+#         cv2.waitKey(1)
+#
+#     if out_test_video:
+#         video.close()
+#
+#     # 测试ms_ssim
+#     if out_test_video:
+#         if video_use_gif:
+#             fps = 0.5
+#             out_wh = (im.shape[1] // 2, im.shape[0] // 2)
+#             suffix = '.gif'
+#         else:
+#             fps = 5
+#             out_wh = (im.shape[1], im.shape[0])
+#             suffix = '.mkv'
+#         video_last_time = time.perf_counter()
+#         video = imageio.get_writer('ms_ssim_test' + suffix, fps=fps)
+#
+#     print('Training MS_SSIM')
+#     rand_im = torch.randint_like(t_im, 0, 255, dtype=torch.float32) / 255.
+#     rand_im.requires_grad = True
+#     optim = torch.optim.Adam([rand_im], 0.003, eps=1e-8)
+#     losser = MS_SSIM(data_range=1., channel=t_im.shape[1]).cuda()
+#     ssim_score = 0
+#     while ssim_score < 0.999:
+#         optim.zero_grad()
+#         loss = losser(rand_im, t_im)
+#         (-loss).sum().backward()
+#         ssim_score = loss.item()
+#         optim.step()
+#         r_im = np.transpose(rand_im.detach().cpu().numpy().clip(0, 1) * 255, [0, 2, 3, 1]).astype(np.uint8)[0]
+#         r_im = cv2.putText(r_im, 'ms_ssim %f' % ssim_score, (10, 30), cv2.FONT_HERSHEY_PLAIN, 2, (255, 0, 0), 2)
+#
+#         if out_test_video:
+#             if time.perf_counter() - video_last_time > 1. / fps:
+#                 video_last_time = time.perf_counter()
+#                 out_frame = cv2.cvtColor(r_im, cv2.COLOR_BGR2RGB)
+#                 out_frame = cv2.resize(out_frame, out_wh, interpolation=cv2.INTER_AREA)
+#                 if isinstance(out_frame, cv2.UMat):
+#                     out_frame = out_frame.get()
+#                 video.append_data(out_frame)
+#
+#         cv2.imshow('ms_ssim', r_im)
+#         cv2.setWindowTitle('ms_ssim', 'ms_ssim %f' % ssim_score)
+#         cv2.waitKey(1)
+#
+#     if out_test_video:
+#         video.close()
+
+"""
+Adapted from https://github.com/Po-Hsun-Su/pytorch-ssim
+"""
+
+import torch
+import torch.nn.functional as F
+from torch.autograd import Variable
+import numpy as np
+from math import exp
+
+
+def gaussian(window_size, sigma):
+    gauss = torch.Tensor([exp(-(x - window_size // 2) ** 2 / float(2 * sigma ** 2)) for x in range(window_size)])
+    return gauss / gauss.sum()
+
+
+def create_window(window_size, channel):
+    _1D_window = gaussian(window_size, 1.5).unsqueeze(1)
+    _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0)
+    window = Variable(_2D_window.expand(channel, 1, window_size, window_size).contiguous())
+    return window
+
+
+def _ssim(img1, img2, window, window_size, channel, size_average=True):
+    mu1 = F.conv2d(img1, window, padding=window_size // 2, groups=channel)
+    mu2 = F.conv2d(img2, window, padding=window_size // 2, groups=channel)
+
+    mu1_sq = mu1.pow(2)
+    mu2_sq = mu2.pow(2)
+    mu1_mu2 = mu1 * mu2
+
+    sigma1_sq = F.conv2d(img1 * img1, window, padding=window_size // 2, groups=channel) - mu1_sq
+    sigma2_sq = F.conv2d(img2 * img2, window, padding=window_size // 2, groups=channel) - mu2_sq
+    sigma12 = F.conv2d(img1 * img2, window, padding=window_size // 2, groups=channel) - mu1_mu2
+
+    C1 = 0.01 ** 2
+    C2 = 0.03 ** 2
+
+    ssim_map = ((2 * mu1_mu2 + C1) * (2 * sigma12 + C2)) / ((mu1_sq + mu2_sq + C1) * (sigma1_sq + sigma2_sq + C2))
+
+    if size_average:
+        return ssim_map.mean()
+    else:
+        return ssim_map.mean(1)
+
+
+class SSIM(torch.nn.Module):
+    def __init__(self, window_size=11, size_average=True):
+        super(SSIM, self).__init__()
+        self.window_size = window_size
+        self.size_average = size_average
+        self.channel = 1
+        self.window = create_window(window_size, self.channel)
+
+    def forward(self, img1, img2):
+        (_, channel, _, _) = img1.size()
+
+        if channel == self.channel and self.window.data.type() == img1.data.type():
+            window = self.window
+        else:
+            window = create_window(self.window_size, channel)
+
+            if img1.is_cuda:
+                window = window.cuda(img1.get_device())
+            window = window.type_as(img1)
+
+            self.window = window
+            self.channel = channel
+
+        return _ssim(img1, img2, window, self.window_size, channel, self.size_average)
+
+
+window = None
+
+
+def ssim(img1, img2, window_size=11, size_average=True):
+    (_, channel, _, _) = img1.size()
+    global window
+    if window is None:
+        window = create_window(window_size, channel)
+        if img1.is_cuda:
+            window = window.cuda(img1.get_device())
+        window = window.type_as(img1)
+    return _ssim(img1, img2, window, window_size, channel, size_average)

modules/fastspeech/fs2.py

@@ -0,0 +1,255 @@
+from modules.commons.common_layers import *
+from modules.commons.common_layers import Embedding
+from modules.fastspeech.tts_modules import FastspeechDecoder, DurationPredictor, LengthRegulator, PitchPredictor, \
+    EnergyPredictor, FastspeechEncoder
+from utils.cwt import cwt2f0
+from utils.hparams import hparams
+from utils.pitch_utils import f0_to_coarse, denorm_f0, norm_f0
+
+FS_ENCODERS = {
+    'fft': lambda hp: FastspeechEncoder(
+        hp['hidden_size'], hp['enc_layers'], hp['enc_ffn_kernel_size'],
+        num_heads=hp['num_heads']),
+}
+
+FS_DECODERS = {
+    'fft': lambda hp: FastspeechDecoder(
+        hp['hidden_size'], hp['dec_layers'], hp['dec_ffn_kernel_size'], hp['num_heads']),
+}
+
+
+class FastSpeech2(nn.Module):
+    def __init__(self, dictionary, out_dims=None):
+        super().__init__()
+        # self.dictionary = dictionary
+        self.padding_idx = 0
+        if not hparams['no_fs2'] if 'no_fs2' in hparams.keys() else True:
+            self.enc_layers = hparams['enc_layers']
+            self.dec_layers = hparams['dec_layers']
+            self.encoder = FS_ENCODERS[hparams['encoder_type']](hparams)
+            self.decoder = FS_DECODERS[hparams['decoder_type']](hparams)
+        self.hidden_size = hparams['hidden_size']
+        # self.encoder_embed_tokens = self.build_embedding(self.dictionary, self.hidden_size)
+        self.out_dims = out_dims
+        if out_dims is None:
+            self.out_dims = hparams['audio_num_mel_bins']
+        self.mel_out = Linear(self.hidden_size, self.out_dims, bias=True)
+        #=========not used===========
+        # if hparams['use_spk_id']:
+        #     self.spk_embed_proj = Embedding(hparams['num_spk'] + 1, self.hidden_size)
+        #     if hparams['use_split_spk_id']:
+        #         self.spk_embed_f0 = Embedding(hparams['num_spk'] + 1, self.hidden_size)
+        #         self.spk_embed_dur = Embedding(hparams['num_spk'] + 1, self.hidden_size)
+        # elif hparams['use_spk_embed']:
+        #     self.spk_embed_proj = Linear(256, self.hidden_size, bias=True)
+        predictor_hidden = hparams['predictor_hidden'] if hparams['predictor_hidden'] > 0 else self.hidden_size
+        # self.dur_predictor = DurationPredictor(
+        #     self.hidden_size,
+        #     n_chans=predictor_hidden,
+        #     n_layers=hparams['dur_predictor_layers'],
+        #     dropout_rate=hparams['predictor_dropout'], padding=hparams['ffn_padding'],
+        #     kernel_size=hparams['dur_predictor_kernel'])
+        # self.length_regulator = LengthRegulator()
+        if hparams['use_pitch_embed']:
+            self.pitch_embed = Embedding(300, self.hidden_size, self.padding_idx)
+            if hparams['pitch_type'] == 'cwt':
+                h = hparams['cwt_hidden_size']
+                cwt_out_dims = 10
+                if hparams['use_uv']:
+                    cwt_out_dims = cwt_out_dims + 1
+                self.cwt_predictor = nn.Sequential(
+                    nn.Linear(self.hidden_size, h),
+                    PitchPredictor(
+                        h,
+                        n_chans=predictor_hidden,
+                        n_layers=hparams['predictor_layers'],
+                        dropout_rate=hparams['predictor_dropout'], odim=cwt_out_dims,
+                        padding=hparams['ffn_padding'], kernel_size=hparams['predictor_kernel']))
+                self.cwt_stats_layers = nn.Sequential(
+                    nn.Linear(self.hidden_size, h), nn.ReLU(),
+                    nn.Linear(h, h), nn.ReLU(), nn.Linear(h, 2)
+                )
+            else:
+                self.pitch_predictor = PitchPredictor(
+                    self.hidden_size,
+                    n_chans=predictor_hidden,
+                    n_layers=hparams['predictor_layers'],
+                    dropout_rate=hparams['predictor_dropout'],
+                    odim=2 if hparams['pitch_type'] == 'frame' else 1,
+                    padding=hparams['ffn_padding'], kernel_size=hparams['predictor_kernel'])
+        if hparams['use_energy_embed']:
+            self.energy_embed = Embedding(256, self.hidden_size, self.padding_idx)
+            # self.energy_predictor = EnergyPredictor(
+            #     self.hidden_size,
+            #     n_chans=predictor_hidden,
+            #     n_layers=hparams['predictor_layers'],
+            #     dropout_rate=hparams['predictor_dropout'], odim=1,
+            #     padding=hparams['ffn_padding'], kernel_size=hparams['predictor_kernel'])
+
+    # def build_embedding(self, dictionary, embed_dim):
+    #     num_embeddings = len(dictionary)
+    #     emb = Embedding(num_embeddings, embed_dim, self.padding_idx)
+    #     return emb
+
+    def forward(self, hubert, mel2ph=None, spk_embed=None,
+                ref_mels=None, f0=None, uv=None, energy=None, skip_decoder=True,
+                spk_embed_dur_id=None, spk_embed_f0_id=None, infer=False, **kwargs):
+        ret = {}
+        if not hparams['no_fs2'] if 'no_fs2' in hparams.keys() else True:
+            encoder_out =self.encoder(hubert)  # [B, T, C]
+        else:
+            encoder_out =hubert
+        src_nonpadding = (hubert!=0).any(-1)[:,:,None]
+
+        # add ref style embed
+        # Not implemented
+        # variance encoder
+        var_embed = 0
+
+        # encoder_out_dur denotes encoder outputs for duration predictor
+        # in speech adaptation, duration predictor use old speaker embedding
+        if hparams['use_spk_embed']:
+            spk_embed_dur = spk_embed_f0 = spk_embed = self.spk_embed_proj(spk_embed)[:, None, :]
+        elif hparams['use_spk_id']:
+            spk_embed_id = spk_embed
+            if spk_embed_dur_id is None:
+                spk_embed_dur_id = spk_embed_id
+            if spk_embed_f0_id is None:
+                spk_embed_f0_id = spk_embed_id
+            spk_embed = self.spk_embed_proj(spk_embed_id)[:, None, :]
+            spk_embed_dur = spk_embed_f0 = spk_embed
+            if hparams['use_split_spk_id']:
+                spk_embed_dur = self.spk_embed_dur(spk_embed_dur_id)[:, None, :]
+                spk_embed_f0 = self.spk_embed_f0(spk_embed_f0_id)[:, None, :]
+        else:
+            spk_embed_dur = spk_embed_f0 = spk_embed = 0
+
+        # add dur
+        # dur_inp = (encoder_out + var_embed + spk_embed_dur) * src_nonpadding
+
+        # mel2ph = self.add_dur(dur_inp, mel2ph, hubert, ret)
+        ret['mel2ph'] = mel2ph
+
+        decoder_inp = F.pad(encoder_out, [0, 0, 1, 0])
+
+        mel2ph_ = mel2ph[..., None].repeat([1, 1, encoder_out.shape[-1]])
+        decoder_inp_origin = decoder_inp = torch.gather(decoder_inp, 1, mel2ph_)  # [B, T, H]
+
+        tgt_nonpadding = (mel2ph > 0).float()[:, :, None]
+
+        # add pitch and energy embed
+        pitch_inp = (decoder_inp_origin + var_embed + spk_embed_f0) * tgt_nonpadding
+        if hparams['use_pitch_embed']:
+            pitch_inp_ph = (encoder_out + var_embed + spk_embed_f0) * src_nonpadding
+            decoder_inp = decoder_inp + self.add_pitch(pitch_inp, f0, uv, mel2ph, ret, encoder_out=pitch_inp_ph)
+        if hparams['use_energy_embed']:
+            decoder_inp = decoder_inp + self.add_energy(pitch_inp, energy, ret)
+
+        ret['decoder_inp'] = decoder_inp = (decoder_inp + spk_embed) * tgt_nonpadding
+        if not hparams['no_fs2'] if 'no_fs2' in hparams.keys() else True:
+            if skip_decoder:
+                return ret
+            ret['mel_out'] = self.run_decoder(decoder_inp, tgt_nonpadding, ret, infer=infer, **kwargs)
+
+        return ret
+
+    def add_dur(self, dur_input, mel2ph, hubert, ret):
+        src_padding = (hubert==0).all(-1)
+        dur_input = dur_input.detach() + hparams['predictor_grad'] * (dur_input - dur_input.detach())
+        if mel2ph is None:
+            dur, xs = self.dur_predictor.inference(dur_input, src_padding)
+            ret['dur'] = xs
+            ret['dur_choice'] = dur
+            mel2ph = self.length_regulator(dur, src_padding).detach()
+        else:
+            ret['dur'] = self.dur_predictor(dur_input, src_padding)
+        ret['mel2ph'] = mel2ph
+        return mel2ph
+
+    def run_decoder(self, decoder_inp, tgt_nonpadding, ret, infer, **kwargs):
+        x = decoder_inp  # [B, T, H]
+        x = self.decoder(x)
+        x = self.mel_out(x)
+        return x * tgt_nonpadding
+
+    def cwt2f0_norm(self, cwt_spec, mean, std, mel2ph):
+        f0 = cwt2f0(cwt_spec, mean, std, hparams['cwt_scales'])
+        f0 = torch.cat(
+            [f0] + [f0[:, -1:]] * (mel2ph.shape[1] - f0.shape[1]), 1)
+        f0_norm = norm_f0(f0, None, hparams)
+        return f0_norm
+
+    def out2mel(self, out):
+        return out
+    
+    def add_pitch(self,decoder_inp, f0, uv, mel2ph, ret, encoder_out=None):
+        # if hparams['pitch_type'] == 'ph':
+        #     pitch_pred_inp = encoder_out.detach() + hparams['predictor_grad'] * (encoder_out - encoder_out.detach())
+        #     pitch_padding = (encoder_out.sum().abs() == 0)
+        #     ret['pitch_pred'] = pitch_pred = self.pitch_predictor(pitch_pred_inp)
+        #     if f0 is None:
+        #         f0 = pitch_pred[:, :, 0]
+        #     ret['f0_denorm'] = f0_denorm = denorm_f0(f0, None, hparams, pitch_padding=pitch_padding)
+        #     pitch = f0_to_coarse(f0_denorm)  # start from 0 [B, T_txt]
+        #     pitch = F.pad(pitch, [1, 0])
+        #     pitch = torch.gather(pitch, 1, mel2ph)  # [B, T_mel]
+        #     pitch_embedding = pitch_embed(pitch)
+        #     return pitch_embedding
+        
+        decoder_inp = decoder_inp.detach() + hparams['predictor_grad'] * (decoder_inp - decoder_inp.detach())
+
+        pitch_padding = (mel2ph == 0)
+
+        # if hparams['pitch_type'] == 'cwt':
+        #     # NOTE: this part of script is *isolated* from other scripts, which means
+        #     #       it may not be compatible with the current version.    
+        #     pass
+        #     # pitch_padding = None
+        #     # ret['cwt'] = cwt_out = self.cwt_predictor(decoder_inp)
+        #     # stats_out = self.cwt_stats_layers(encoder_out[:, 0, :])  # [B, 2]
+        #     # mean = ret['f0_mean'] = stats_out[:, 0]
+        #     # std = ret['f0_std'] = stats_out[:, 1]
+        #     # cwt_spec = cwt_out[:, :, :10]
+        #     # if f0 is None:
+        #     #     std = std * hparams['cwt_std_scale']
+        #     #     f0 = self.cwt2f0_norm(cwt_spec, mean, std, mel2ph)
+        #     #     if hparams['use_uv']:
+        #     #         assert cwt_out.shape[-1] == 11
+        #     #         uv = cwt_out[:, :, -1] > 0
+        # elif hparams['pitch_ar']:
+        #     ret['pitch_pred'] = pitch_pred = self.pitch_predictor(decoder_inp, f0 if is_training else None)
+        #     if f0 is None:
+        #         f0 = pitch_pred[:, :, 0]
+        # else:
+        #ret['pitch_pred'] = pitch_pred = self.pitch_predictor(decoder_inp)
+        # if f0 is None:
+        #     f0 = pitch_pred[:, :, 0]
+        # if hparams['use_uv'] and uv is None:
+        #     uv = pitch_pred[:, :, 1] > 0
+        ret['f0_denorm'] = f0_denorm = denorm_f0(f0, uv, hparams, pitch_padding=pitch_padding)
+        if pitch_padding is not None:
+            f0[pitch_padding] = 0
+        
+        pitch = f0_to_coarse(f0_denorm,hparams)  # start from 0
+        ret['pitch_pred']=pitch.unsqueeze(-1)
+        # print(ret['pitch_pred'].shape)
+        # print(pitch.shape)
+        pitch_embedding = self.pitch_embed(pitch)
+        return pitch_embedding
+
+    def add_energy(self,decoder_inp, energy, ret):
+        decoder_inp = decoder_inp.detach() + hparams['predictor_grad'] * (decoder_inp - decoder_inp.detach())
+        ret['energy_pred'] = energy#energy_pred = self.energy_predictor(decoder_inp)[:, :, 0]
+        # if energy is None:
+        #     energy = energy_pred
+        energy = torch.clamp(energy * 256 // 4, max=255).long() # energy_to_coarse
+        energy_embedding = self.energy_embed(energy)
+        return energy_embedding
+
+    @staticmethod
+    def mel_norm(x):
+        return (x + 5.5) / (6.3 / 2) - 1
+
+    @staticmethod
+    def mel_denorm(x):
+        return (x + 1) * (6.3 / 2) - 5.5

modules/fastspeech/pe.py

@@ -0,0 +1,149 @@
+from modules.commons.common_layers import *
+from utils.hparams import hparams
+from modules.fastspeech.tts_modules import PitchPredictor
+from utils.pitch_utils import denorm_f0
+
+
+class Prenet(nn.Module):
+    def __init__(self, in_dim=80, out_dim=256, kernel=5, n_layers=3, strides=None):
+        super(Prenet, self).__init__()
+        padding = kernel // 2
+        self.layers = []
+        self.strides = strides if strides is not None else [1] * n_layers
+        for l in range(n_layers):
+            self.layers.append(nn.Sequential(
+                nn.Conv1d(in_dim, out_dim, kernel_size=kernel, padding=padding, stride=self.strides[l]),
+                nn.ReLU(),
+                nn.BatchNorm1d(out_dim)
+            ))
+            in_dim = out_dim
+        self.layers = nn.ModuleList(self.layers)
+        self.out_proj = nn.Linear(out_dim, out_dim)
+
+    def forward(self, x):
+        """
+
+        :param x: [B, T, 80]
+        :return: [L, B, T, H], [B, T, H]
+        """
+        # padding_mask = x.abs().sum(-1).eq(0).data  # [B, T]
+        padding_mask = x.abs().sum(-1).eq(0).detach()
+        nonpadding_mask_TB = 1 - padding_mask.float()[:, None, :]  # [B, 1, T]
+        x = x.transpose(1, 2)
+        hiddens = []
+        for i, l in enumerate(self.layers):
+            nonpadding_mask_TB = nonpadding_mask_TB[:, :, ::self.strides[i]]
+            x = l(x) * nonpadding_mask_TB
+        hiddens.append(x)
+        hiddens = torch.stack(hiddens, 0)  # [L, B, H, T]
+        hiddens = hiddens.transpose(2, 3)  # [L, B, T, H]
+        x = self.out_proj(x.transpose(1, 2))  # [B, T, H]
+        x = x * nonpadding_mask_TB.transpose(1, 2)
+        return hiddens, x
+
+
+class ConvBlock(nn.Module):
+    def __init__(self, idim=80, n_chans=256, kernel_size=3, stride=1, norm='gn', dropout=0):
+        super().__init__()
+        self.conv = ConvNorm(idim, n_chans, kernel_size, stride=stride)
+        self.norm = norm
+        if self.norm == 'bn':
+            self.norm = nn.BatchNorm1d(n_chans)
+        elif self.norm == 'in':
+            self.norm = nn.InstanceNorm1d(n_chans, affine=True)
+        elif self.norm == 'gn':
+            self.norm = nn.GroupNorm(n_chans // 16, n_chans)
+        elif self.norm == 'ln':
+            self.norm = LayerNorm(n_chans // 16, n_chans)
+        elif self.norm == 'wn':
+            self.conv = torch.nn.utils.weight_norm(self.conv.conv)
+        self.dropout = nn.Dropout(dropout)
+        self.relu = nn.ReLU()
+
+    def forward(self, x):
+        """
+
+        :param x: [B, C, T]
+        :return: [B, C, T]
+        """
+        x = self.conv(x)
+        if not isinstance(self.norm, str):
+            if self.norm == 'none':
+                pass
+            elif self.norm == 'ln':
+                x = self.norm(x.transpose(1, 2)).transpose(1, 2)
+            else:
+                x = self.norm(x)
+        x = self.relu(x)
+        x = self.dropout(x)
+        return x
+
+
+class ConvStacks(nn.Module):
+    def __init__(self, idim=80, n_layers=5, n_chans=256, odim=32, kernel_size=5, norm='gn',
+                 dropout=0, strides=None, res=True):
+        super().__init__()
+        self.conv = torch.nn.ModuleList()
+        self.kernel_size = kernel_size
+        self.res = res
+        self.in_proj = Linear(idim, n_chans)
+        if strides is None:
+            strides = [1] * n_layers
+        else:
+            assert len(strides) == n_layers
+        for idx in range(n_layers):
+            self.conv.append(ConvBlock(
+                n_chans, n_chans, kernel_size, stride=strides[idx], norm=norm, dropout=dropout))
+        self.out_proj = Linear(n_chans, odim)
+
+    def forward(self, x, return_hiddens=False):
+        """
+
+        :param x: [B, T, H]
+        :return: [B, T, H]
+        """
+        x = self.in_proj(x)
+        x = x.transpose(1, -1)  # (B, idim, Tmax)
+        hiddens = []
+        for f in self.conv:
+            x_ = f(x)
+            x = x + x_ if self.res else x_  # (B, C, Tmax)
+            hiddens.append(x)
+        x = x.transpose(1, -1)
+        x = self.out_proj(x)  # (B, Tmax, H)
+        if return_hiddens:
+            hiddens = torch.stack(hiddens, 1)  # [B, L, C, T]
+            return x, hiddens
+        return x
+
+
+class PitchExtractor(nn.Module):
+    def __init__(self, n_mel_bins=80, conv_layers=2):
+        super().__init__()
+        self.hidden_size = hparams['hidden_size']
+        self.predictor_hidden = hparams['predictor_hidden'] if hparams['predictor_hidden'] > 0 else self.hidden_size
+        self.conv_layers = conv_layers
+
+        self.mel_prenet = Prenet(n_mel_bins, self.hidden_size, strides=[1, 1, 1])
+        if self.conv_layers > 0:
+            self.mel_encoder = ConvStacks(
+                    idim=self.hidden_size, n_chans=self.hidden_size, odim=self.hidden_size, n_layers=self.conv_layers)
+        self.pitch_predictor = PitchPredictor(
+            self.hidden_size, n_chans=self.predictor_hidden,
+            n_layers=5, dropout_rate=0.1, odim=2,
+            padding=hparams['ffn_padding'], kernel_size=hparams['predictor_kernel'])
+
+    def forward(self, mel_input=None):
+        ret = {}
+        mel_hidden = self.mel_prenet(mel_input)[1]
+        if self.conv_layers > 0:
+            mel_hidden = self.mel_encoder(mel_hidden)
+
+        ret['pitch_pred'] = pitch_pred = self.pitch_predictor(mel_hidden)
+
+        pitch_padding = mel_input.abs().sum(-1) == 0
+        use_uv = hparams['pitch_type'] == 'frame' #and hparams['use_uv']
+        ret['f0_denorm_pred'] = denorm_f0(
+            pitch_pred[:, :, 0], (pitch_pred[:, :, 1] > 0) if use_uv else None,
+            hparams, pitch_padding=pitch_padding)
+        return ret

modules/fastspeech/tts_modules.py

@@ -0,0 +1,364 @@
+import logging
+import math
+
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+
+from modules.commons.espnet_positional_embedding import RelPositionalEncoding
+from modules.commons.common_layers import SinusoidalPositionalEmbedding, Linear, EncSALayer, DecSALayer, BatchNorm1dTBC
+from utils.hparams import hparams
+
+DEFAULT_MAX_SOURCE_POSITIONS = 2000
+DEFAULT_MAX_TARGET_POSITIONS = 2000
+
+
+class TransformerEncoderLayer(nn.Module):
+    def __init__(self, hidden_size, dropout, kernel_size=None, num_heads=2, norm='ln'):
+        super().__init__()
+        self.hidden_size = hidden_size
+        self.dropout = dropout
+        self.num_heads = num_heads
+        self.op = EncSALayer(
+            hidden_size, num_heads, dropout=dropout,
+            attention_dropout=0.0, relu_dropout=dropout,
+            kernel_size=kernel_size
+            if kernel_size is not None else hparams['enc_ffn_kernel_size'],
+            padding=hparams['ffn_padding'],
+            norm=norm, act=hparams['ffn_act'])
+
+    def forward(self, x, **kwargs):
+        return self.op(x, **kwargs)
+
+
+######################
+# fastspeech modules
+######################
+class LayerNorm(torch.nn.LayerNorm):
+    """Layer normalization module.
+    :param int nout: output dim size
+    :param int dim: dimension to be normalized
+    """
+
+    def __init__(self, nout, dim=-1):
+        """Construct an LayerNorm object."""
+        super(LayerNorm, self).__init__(nout, eps=1e-12)
+        self.dim = dim
+
+    def forward(self, x):
+        """Apply layer normalization.
+        :param torch.Tensor x: input tensor
+        :return: layer normalized tensor
+        :rtype torch.Tensor
+        """
+        if self.dim == -1:
+            return super(LayerNorm, self).forward(x)
+        return super(LayerNorm, self).forward(x.transpose(1, -1)).transpose(1, -1)
+
+
+class DurationPredictor(torch.nn.Module):
+    """Duration predictor module.
+    This is a module of duration predictor described in `FastSpeech: Fast, Robust and Controllable Text to Speech`_.
+    The duration predictor predicts a duration of each frame in log domain from the hidden embeddings of encoder.
+    .. _`FastSpeech: Fast, Robust and Controllable Text to Speech`:
+        https://arxiv.org/pdf/1905.09263.pdf
+    Note:
+        The calculation domain of outputs is different between in `forward` and in `inference`. In `forward`,
+        the outputs are calculated in log domain but in `inference`, those are calculated in linear domain.
+    """
+
+    def __init__(self, idim, n_layers=2, n_chans=384, kernel_size=3, dropout_rate=0.1, offset=1.0, padding='SAME'):
+        """Initilize duration predictor module.
+        Args:
+            idim (int): Input dimension.
+            n_layers (int, optional): Number of convolutional layers.
+            n_chans (int, optional): Number of channels of convolutional layers.
+            kernel_size (int, optional): Kernel size of convolutional layers.
+            dropout_rate (float, optional): Dropout rate.
+            offset (float, optional): Offset value to avoid nan in log domain.
+        """
+        super(DurationPredictor, self).__init__()
+        self.offset = offset
+        self.conv = torch.nn.ModuleList()
+        self.kernel_size = kernel_size
+        self.padding = padding
+        for idx in range(n_layers):
+            in_chans = idim if idx == 0 else n_chans
+            self.conv += [torch.nn.Sequential(
+                torch.nn.ConstantPad1d(((kernel_size - 1) // 2, (kernel_size - 1) // 2)
+                                       if padding == 'SAME'
+                                       else (kernel_size - 1, 0), 0),
+                torch.nn.Conv1d(in_chans, n_chans, kernel_size, stride=1, padding=0),
+                torch.nn.ReLU(),
+                LayerNorm(n_chans, dim=1),
+                torch.nn.Dropout(dropout_rate)
+            )]
+        if hparams['dur_loss'] in ['mse', 'huber']:
+            odims = 1
+        elif hparams['dur_loss'] == 'mog':
+            odims = 15
+        elif hparams['dur_loss'] == 'crf':
+            odims = 32
+            from torchcrf import CRF
+            self.crf = CRF(odims, batch_first=True)
+        self.linear = torch.nn.Linear(n_chans, odims)
+
+    def _forward(self, xs, x_masks=None, is_inference=False):
+        xs = xs.transpose(1, -1)  # (B, idim, Tmax)
+        for f in self.conv:
+            xs = f(xs)  # (B, C, Tmax)
+            if x_masks is not None:
+                xs = xs * (1 - x_masks.float())[:, None, :]
+
+        xs = self.linear(xs.transpose(1, -1))  # [B, T, C]
+        xs = xs * (1 - x_masks.float())[:, :, None]  # (B, T, C)
+        if is_inference:
+            return self.out2dur(xs), xs
+        else:
+            if hparams['dur_loss'] in ['mse']:
+                xs = xs.squeeze(-1)  # (B, Tmax)
+        return xs
+
+    def out2dur(self, xs):
+        if hparams['dur_loss'] in ['mse']:
+            # NOTE: calculate in log domain
+            xs = xs.squeeze(-1)  # (B, Tmax)
+            dur = torch.clamp(torch.round(xs.exp() - self.offset), min=0).long()  # avoid negative value
+        elif hparams['dur_loss'] == 'mog':
+            return NotImplementedError
+        elif hparams['dur_loss'] == 'crf':
+            dur = torch.LongTensor(self.crf.decode(xs)).cuda()
+        return dur
+
+    def forward(self, xs, x_masks=None):
+        """Calculate forward propagation.
+        Args:
+            xs (Tensor): Batch of input sequences (B, Tmax, idim).
+            x_masks (ByteTensor, optional): Batch of masks indicating padded part (B, Tmax).
+        Returns:
+            Tensor: Batch of predicted durations in log domain (B, Tmax).
+        """
+        return self._forward(xs, x_masks, False)
+
+    def inference(self, xs, x_masks=None):
+        """Inference duration.
+        Args:
+            xs (Tensor): Batch of input sequences (B, Tmax, idim).
+            x_masks (ByteTensor, optional): Batch of masks indicating padded part (B, Tmax).
+        Returns:
+            LongTensor: Batch of predicted durations in linear domain (B, Tmax).
+        """
+        return self._forward(xs, x_masks, True)
+
+
+class LengthRegulator(torch.nn.Module):
+    def __init__(self, pad_value=0.0):
+        super(LengthRegulator, self).__init__()
+        self.pad_value = pad_value
+
+    def forward(self, dur, dur_padding=None, alpha=1.0):
+        """
+        Example (no batch dim version):
+            1. dur = [2,2,3]
+            2. token_idx = [[1],[2],[3]], dur_cumsum = [2,4,7], dur_cumsum_prev = [0,2,4]
+            3. token_mask = [[1,1,0,0,0,0,0],
+                             [0,0,1,1,0,0,0],
+                             [0,0,0,0,1,1,1]]
+            4. token_idx * token_mask = [[1,1,0,0,0,0,0],
+                                         [0,0,2,2,0,0,0],
+                                         [0,0,0,0,3,3,3]]
+            5. (token_idx * token_mask).sum(0) = [1,1,2,2,3,3,3]
+
+        :param dur: Batch of durations of each frame (B, T_txt)
+        :param dur_padding: Batch of padding of each frame (B, T_txt)
+        :param alpha: duration rescale coefficient
+        :return:
+            mel2ph (B, T_speech)
+        """
+        assert alpha > 0
+        dur = torch.round(dur.float() * alpha).long()
+        if dur_padding is not None:
+            dur = dur * (1 - dur_padding.long())
+        token_idx = torch.arange(1, dur.shape[1] + 1)[None, :, None].to(dur.device)
+        dur_cumsum = torch.cumsum(dur, 1)
+        dur_cumsum_prev = F.pad(dur_cumsum, [1, -1], mode='constant', value=0)
+
+        pos_idx = torch.arange(dur.sum(-1).max())[None, None].to(dur.device)
+        token_mask = (pos_idx >= dur_cumsum_prev[:, :, None]) & (pos_idx < dur_cumsum[:, :, None])
+        mel2ph = (token_idx * token_mask.long()).sum(1)
+        return mel2ph
+
+
+class PitchPredictor(torch.nn.Module):
+    def __init__(self, idim, n_layers=5, n_chans=384, odim=2, kernel_size=5,
+                 dropout_rate=0.1, padding='SAME'):
+        """Initilize pitch predictor module.
+        Args:
+            idim (int): Input dimension.
+            n_layers (int, optional): Number of convolutional layers.
+            n_chans (int, optional): Number of channels of convolutional layers.
+            kernel_size (int, optional): Kernel size of convolutional layers.
+            dropout_rate (float, optional): Dropout rate.
+        """
+        super(PitchPredictor, self).__init__()
+        self.conv = torch.nn.ModuleList()
+        self.kernel_size = kernel_size
+        self.padding = padding
+        for idx in range(n_layers):
+            in_chans = idim if idx == 0 else n_chans
+            self.conv += [torch.nn.Sequential(
+                torch.nn.ConstantPad1d(((kernel_size - 1) // 2, (kernel_size - 1) // 2)
+                                       if padding == 'SAME'
+                                       else (kernel_size - 1, 0), 0),
+                torch.nn.Conv1d(in_chans, n_chans, kernel_size, stride=1, padding=0),
+                torch.nn.ReLU(),
+                LayerNorm(n_chans, dim=1),
+                torch.nn.Dropout(dropout_rate)
+            )]
+        self.linear = torch.nn.Linear(n_chans, odim)
+        self.embed_positions = SinusoidalPositionalEmbedding(idim, 0, init_size=4096)
+        self.pos_embed_alpha = nn.Parameter(torch.Tensor([1]))
+
+    def forward(self, xs):
+        """
+
+        :param xs: [B, T, H]
+        :return: [B, T, H]
+        """
+        positions = self.pos_embed_alpha * self.embed_positions(xs[..., 0])
+        xs = xs + positions
+        xs = xs.transpose(1, -1)  # (B, idim, Tmax)
+        for f in self.conv:
+            xs = f(xs)  # (B, C, Tmax)
+        # NOTE: calculate in log domain
+        xs = self.linear(xs.transpose(1, -1))  # (B, Tmax, H)
+        return xs
+
+
+class EnergyPredictor(PitchPredictor):
+    pass
+
+
+def mel2ph_to_dur(mel2ph, T_txt, max_dur=None):
+    B, _ = mel2ph.shape
+    dur = mel2ph.new_zeros(B, T_txt + 1).scatter_add(1, mel2ph, torch.ones_like(mel2ph))
+    dur = dur[:, 1:]
+    if max_dur is not None:
+        dur = dur.clamp(max=max_dur)
+    return dur
+
+
+class FFTBlocks(nn.Module):
+    def __init__(self, hidden_size, num_layers, ffn_kernel_size=9, dropout=None, num_heads=2,
+                 use_pos_embed=True, use_last_norm=True, norm='ln', use_pos_embed_alpha=True):
+        super().__init__()
+        self.num_layers = num_layers
+        embed_dim = self.hidden_size = hidden_size
+        self.dropout = dropout if dropout is not None else hparams['dropout']
+        self.use_pos_embed = use_pos_embed
+        self.use_last_norm = use_last_norm
+        if use_pos_embed:
+            self.max_source_positions = DEFAULT_MAX_TARGET_POSITIONS
+            self.padding_idx = 0
+            self.pos_embed_alpha = nn.Parameter(torch.Tensor([1])) if use_pos_embed_alpha else 1
+            self.embed_positions = SinusoidalPositionalEmbedding(
+                embed_dim, self.padding_idx, init_size=DEFAULT_MAX_TARGET_POSITIONS,
+            )
+
+        self.layers = nn.ModuleList([])
+        self.layers.extend([
+            TransformerEncoderLayer(self.hidden_size, self.dropout,
+                                    kernel_size=ffn_kernel_size, num_heads=num_heads)
+            for _ in range(self.num_layers)
+        ])
+        if self.use_last_norm:
+            if norm == 'ln':
+                self.layer_norm = nn.LayerNorm(embed_dim)
+            elif norm == 'bn':
+                self.layer_norm = BatchNorm1dTBC(embed_dim)
+        else:
+            self.layer_norm = None
+
+    def forward(self, x, padding_mask=None, attn_mask=None, return_hiddens=False):
+        """
+        :param x: [B, T, C]
+        :param padding_mask: [B, T]
+        :return: [B, T, C] or [L, B, T, C]
+        """
+        # padding_mask = x.abs().sum(-1).eq(0).data if padding_mask is None else padding_mask
+        padding_mask = x.abs().sum(-1).eq(0).detach() if padding_mask is None else padding_mask
+        nonpadding_mask_TB = 1 - padding_mask.transpose(0, 1).float()[:, :, None]  # [T, B, 1]
+        if self.use_pos_embed:
+            positions = self.pos_embed_alpha * self.embed_positions(x[..., 0])
+            x = x + positions
+            x = F.dropout(x, p=self.dropout, training=self.training)
+        # B x T x C -> T x B x C
+        x = x.transpose(0, 1) * nonpadding_mask_TB
+        hiddens = []
+        for layer in self.layers:
+            x = layer(x, encoder_padding_mask=padding_mask, attn_mask=attn_mask) * nonpadding_mask_TB
+            hiddens.append(x)
+        if self.use_last_norm:
+            x = self.layer_norm(x) * nonpadding_mask_TB
+        if return_hiddens:
+            x = torch.stack(hiddens, 0)  # [L, T, B, C]
+            x = x.transpose(1, 2)  # [L, B, T, C]
+        else:
+            x = x.transpose(0, 1)  # [B, T, C]
+        return x
+
+
+class FastspeechEncoder(FFTBlocks):
+    '''
+        compared to FFTBlocks:
+        - input is [B, T, H], not [B, T, C]
+        - supports "relative" positional encoding
+    '''
+    def __init__(self, hidden_size=None, num_layers=None, kernel_size=None, num_heads=2):
+        hidden_size = hparams['hidden_size'] if hidden_size is None else hidden_size
+        kernel_size = hparams['enc_ffn_kernel_size'] if kernel_size is None else kernel_size
+        num_layers = hparams['dec_layers'] if num_layers is None else num_layers
+        super().__init__(hidden_size, num_layers, kernel_size, num_heads=num_heads,
+                         use_pos_embed=False)  # use_pos_embed_alpha for compatibility
+        #self.embed_tokens = embed_tokens
+        self.embed_scale = math.sqrt(hidden_size)
+        self.padding_idx = 0
+        if hparams.get('rel_pos') is not None and hparams['rel_pos']:
+            self.embed_positions = RelPositionalEncoding(hidden_size, dropout_rate=0.0)
+        else:
+            self.embed_positions = SinusoidalPositionalEmbedding(
+                hidden_size, self.padding_idx, init_size=DEFAULT_MAX_TARGET_POSITIONS,
+            )
+
+    def forward(self, hubert):
+        """
+
+        :param hubert: [B, T, H ]
+        :return: {
+            'encoder_out': [T x B x C]
+        }
+        """
+        # encoder_padding_mask = txt_tokens.eq(self.padding_idx).data
+        encoder_padding_mask = (hubert==0).all(-1)
+        x = self.forward_embedding(hubert)  # [B, T, H]
+        x = super(FastspeechEncoder, self).forward(x, encoder_padding_mask)
+        return x
+
+    def forward_embedding(self, hubert):
+        # embed tokens and positions
+        x = self.embed_scale * hubert
+        if hparams['use_pos_embed']:
+            positions = self.embed_positions(hubert)
+            x = x + positions
+        x = F.dropout(x, p=self.dropout, training=self.training)
+        return x
+
+
+class FastspeechDecoder(FFTBlocks):
+    def __init__(self, hidden_size=None, num_layers=None, kernel_size=None, num_heads=None):
+        num_heads = hparams['num_heads'] if num_heads is None else num_heads
+        hidden_size = hparams['hidden_size'] if hidden_size is None else hidden_size
+        kernel_size = hparams['dec_ffn_kernel_size'] if kernel_size is None else kernel_size
+        num_layers = hparams['dec_layers'] if num_layers is None else num_layers
+        super().__init__(hidden_size, num_layers, kernel_size, num_heads=num_heads)
+